--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:43:36.311483Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-0213e-9d9695dfb-m2mzx to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:43:36Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c553b5401694 namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 1b6b4d08-4f1e-4dc5-bf33-815796d0db14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:43:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-0213e-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:37Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c553c3721e87 namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 8abf600a-85a1-437d-ab6d-ed9de258729a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:38Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:43:38Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:43:38Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c55427bd66d9 namespace: kserve-ci-e2e-test resourceVersion: "19600" uid: 054c5799-2354-4578-bc3e-1aab38a8a4c9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0213e} kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:43:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:38Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c55428f1066a namespace: kserve-ci-e2e-test resourceVersion: "19602" uid: 260774a3-de6e-404a-9d58-bc2966a1b61d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0213e} kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:43:38Z" message: 'Created container: ensemble-graph-0213e' metadata: creationTimestamp: "2026-04-22T19:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:38Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c5542c51d0ca namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: 6849901f-828b-47e2-8d3d-956e97be38fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0213e} kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:43:38Z" message: Started container ensemble-graph-0213e metadata: creationTimestamp: "2026-04-22T19:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:38Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c5542cbf1f3b namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 4f80df4d-2581-4ec2-8331-3a24cc72e64b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0213e} kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Stopping container ensemble-graph-0213e metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c5560d7edc10 namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: b0549b2a-51ac-477c-9ed4-1732899eb03d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:43:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0213e} kind: Pod name: ensemble-graph-0213e-9d9695dfb-m2mzx namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 9d2a5a4c-c2b9-4d78-ba2a-0c5cebeeed61 kind: Event lastTimestamp: "2026-04-22T19:44:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:43:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:15Z" name: ensemble-graph-0213e-9d9695dfb-m2mzx.18a8c556eb7ca659 namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: f730e145-724f-4eeb-aef4-3d1d6412bfdf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-0213e-9d9695dfb namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 4541bf64-1aa9-4fb7-864b-5e28db055f44 kind: Event lastTimestamp: "2026-04-22T19:43:36Z" message: 'Created pod: ensemble-graph-0213e-9d9695dfb-m2mzx' metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:36Z" name: ensemble-graph-0213e-9d9695dfb.18a8c553b4d0eb19 namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: acbbd16f-7bec-4615-99f2-3813381fd071 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-0213e namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 8647865a-e7a5-4fc6-b1f6-0d8024900801 kind: Event lastTimestamp: "2026-04-22T19:43:36Z" message: Scaled up replica set ensemble-graph-0213e-9d9695dfb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:36Z" name: ensemble-graph-0213e.18a8c553b4050047 namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: 944b2137-ed21-42ca-b5ed-839a377975ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0213e namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 40e626f9-9769-4e14-aa96-a08a23e66252 kind: Event lastTimestamp: "2026-04-22T19:43:36Z" message: InferenceGraph [ensemble-graph-0213e] is Ready metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:36Z" name: ensemble-graph-0213e.18a8c553b9100d0a namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: 94e04f7f-e485-4c33-86d0-1fe4a54d6dd7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0213e namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 40e626f9-9769-4e14-aa96-a08a23e66252 kind: Event lastTimestamp: "2026-04-22T19:43:36Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-0213e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-0213e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:36Z" name: ensemble-graph-0213e.18a8c553bb45f611 namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 46bfe240-1cb3-4aaa-9da0-1108bb7bf6ac reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0213e namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 40e626f9-9769-4e14-aa96-a08a23e66252 kind: Event lastTimestamp: "2026-04-22T19:43:36Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-0213e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:36Z" name: ensemble-graph-0213e.18a8c553bb464c35 namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: b8ca6d4e-a4d9-49e2-a49f-4b94f7bd212f reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:56.616878Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-62743-7bf7944f9c-pj267 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:56Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c56667d31b78 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 2ef68831-bf9c-41fe-a2f3-73eb7e01d3b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:57Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c56680d5259f namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 4caf7526-88af-4806-ac21-4d01b4af9310 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-62743} kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:57Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c566821a892c namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 8d53749b-3cf5-4e43-8f55-feb7f5b15583 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-62743} kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: 'Created container: ensemble-graph-62743' metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:57Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c56685c81ae4 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 45af9c68-2e69-4806-9249-d5f11ce96f00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-62743} kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: Started container ensemble-graph-62743 metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:57Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c566863a694e namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 81f2f241-0033-4af6-a1ca-31ac6ec853ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-62743} kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Stopping container ensemble-graph-62743 metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c5d997caeeea namespace: kserve-ci-e2e-test resourceVersion: "24100" uid: 873ceec8-fdcf-4dc8-8aa6-d6566f759e0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:53:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-62743} kind: Pod name: ensemble-graph-62743-7bf7944f9c-pj267 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 055d9073-bc43-43f7-8fc6-d3c781cf6c85 kind: Event lastTimestamp: "2026-04-22T19:53:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:38Z" name: ensemble-graph-62743-7bf7944f9c-pj267.18a8c5da0ff56feb namespace: kserve-ci-e2e-test resourceVersion: "24414" uid: 9a074482-fa8d-4d14-a1ff-aee19af8c8ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-62743-7bf7944f9c namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: ceec24ea-da51-4afa-9616-2e0c77c0ff64 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: 'Created pod: ensemble-graph-62743-7bf7944f9c-pj267' metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:56Z" name: ensemble-graph-62743-7bf7944f9c.18a8c566675c9322 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 1c0370f8-8375-447f-a466-9b265fecd2c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-62743 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 349c11ce-7cad-42de-9c16-598cef858b30 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: Scaled up replica set ensemble-graph-62743-7bf7944f9c from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:56Z" name: ensemble-graph-62743.18a8c56666a1b097 namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 6eed744a-f564-4377-9ba3-b2bd407d1a90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-62743 namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: dfe164cc-f267-4fca-8534-822b10092bf9 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: InferenceGraph [ensemble-graph-62743] is Ready metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:56Z" name: ensemble-graph-62743.18a8c5666af88687 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: da98b179-1048-49be-930e-02d408c0a1d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-62743 namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: 4f470389-414f-4678-a491-0eb8b7d22df2 kind: Event lastTimestamp: "2026-04-22T19:45:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:26Z" name: ensemble-graph-62743.18a8c569e50d1d50 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 69522692-60fe-484e-ae86-0f9cf76c3876 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-62743 namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: 4f470389-414f-4678-a491-0eb8b7d22df2 kind: Event lastTimestamp: "2026-04-22T19:45:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:26Z" name: ensemble-graph-62743.18a8c569e56e150a namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 297a8307-b8c6-4b62-a63c-d4b03957c677 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:42:26.262879Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18832" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0213e-predictor-6bb556f566-44hqn to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c543660725cb namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: ba8eaf0b-d8e0-4fa6-902c-7952617fac90 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c5436dd0ffc1 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 0f69b3e6-f8eb-41ad-8278-84a69ff90b1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c5436f16dc81 namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: ba9214f6-c138-46d2-bc53-e13caa242cfc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c543746eb283 namespace: kserve-ci-e2e-test resourceVersion: "18857" uid: c7d6fb17-0062-46fc-9491-9c6849d71ff6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c54374e0fc10 namespace: kserve-ci-e2e-test resourceVersion: "18858" uid: 73c9125e-39d7-42b1-b236-463b088751b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: "2026-04-22T19:43:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:48Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c54393ffb7ca namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: a92bebd6-a434-4126-9878-4ed46eefe1f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0213e-predictor-6bb556f566-44hqn namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 3c692a3c-7b5a-406c-a638-c87a2a3d9c19 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-0213e-predictor-6bb556f566-44hqn.18a8c55616840059 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 1fd99e5e-0718-4254-8689-ec41f0a7f181 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0213e-predictor-6bb556f566 namespace: kserve-ci-e2e-test resourceVersion: "18821" uid: 502cfe3b-c831-4213-884a-86b4e8b38b46 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: 'Created pod: error-404-isvc-0213e-predictor-6bb556f566-44hqn' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor-6bb556f566.18a8c54365a8e9a0 namespace: kserve-ci-e2e-test resourceVersion: "18833" uid: 04e1317d-e3d0-4f00-8e94-77272e3b8305 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18817" uid: f17138ca-a6dd-4815-bd9b-f7cdc4de476f kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Scaled up replica set error-404-isvc-0213e-predictor-6bb556f566 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e-predictor.18a8c54364368963 namespace: kserve-ci-e2e-test resourceVersion: "18825" uid: 65e065ec-c6d8-4ecd-ad15-f9d377355524 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18820" uid: c1bb19db-7322-4393-90a9-67e0a0ca19ec kind: Event lastTimestamp: "2026-04-22T19:42:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:56Z" name: error-404-isvc-0213e-predictor.18a8c546e2ade802 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 4f6b3c9e-f7fe-4790-9644-a6a2cf1aab64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18820" uid: c1bb19db-7322-4393-90a9-67e0a0ca19ec kind: Event lastTimestamp: "2026-04-22T19:42:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:56Z" name: error-404-isvc-0213e-predictor.18a8c546e30f515d namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: e40811f1-6d78-4ce9-94d5-28e74f336b1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:43:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18984" uid: c1bb19db-7322-4393-90a9-67e0a0ca19ec kind: Event lastTimestamp: "2026-04-22T19:43:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:41Z" name: error-404-isvc-0213e-predictor.18a8c54de04c62e1 namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 121dbc26-26d6-46dc-9444-c491a682f126 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:43:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18984" uid: c1bb19db-7322-4393-90a9-67e0a0ca19ec kind: Event lastTimestamp: "2026-04-22T19:43:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:41Z" name: error-404-isvc-0213e-predictor.18a8c54de0abf55c namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: 9e9261cf-ad5d-48ce-9db4-5a5f92b2596c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0213e namespace: kserve-ci-e2e-test resourceVersion: "18804" uid: 11e84918-51c8-418b-91ae-79f3ec745151 kind: Event lastTimestamp: "2026-04-22T19:43:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:18Z" name: error-404-isvc-0213e.18a8c5436410fef4 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 63a7feac-fe11-489c-be4c-ce4688a39c81 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0213e namespace: kserve-ci-e2e-test resourceVersion: "18804" uid: 11e84918-51c8-418b-91ae-79f3ec745151 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-0213e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0213e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e.18a8c543649a6066 namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 371deef7-fdcb-46b3-aea3-25bbadb53265 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0213e namespace: kserve-ci-e2e-test resourceVersion: "18804" uid: 11e84918-51c8-418b-91ae-79f3ec745151 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0213e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-0213e.18a8c543649ac08b namespace: kserve-ci-e2e-test resourceVersion: "18829" uid: 0be124aa-d1b6-4a56-9cec-76ba73419eb4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0213e namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: 11e84918-51c8-418b-91ae-79f3ec745151 kind: Event lastTimestamp: "2026-04-22T19:43:18Z" message: InferenceService [error-404-isvc-0213e] is Ready metadata: creationTimestamp: "2026-04-22T19:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:18Z" name: error-404-isvc-0213e.18a8c54f776a2e87 namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: e4e8c832-16a2-4fce-90eb-202b3acf4744 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:54:31.858233Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25113" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:54:31Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c5ec56e197fb namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: fbcb1afe-0bbe-49c6-9d40-3460a3017519 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: "2026-04-22T19:54:32Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:54:32Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c5ec5fb534a0 namespace: kserve-ci-e2e-test resourceVersion: "25136" uid: 34563a92-2e6d-43d0-8c62-bc79c39edb13 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: "2026-04-22T19:54:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:32Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c5ec60ff589e namespace: kserve-ci-e2e-test resourceVersion: "25138" uid: e79aeea9-90f8-4979-951f-39fbbb32035f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: "2026-04-22T19:54:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:32Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c5ec6599a6c1 namespace: kserve-ci-e2e-test resourceVersion: "25143" uid: 0cebda6c-277a-4983-b5cf-4287be56e246 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: "2026-04-22T19:54:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:32Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c5ec66198d88 namespace: kserve-ci-e2e-test resourceVersion: "25145" uid: ca622d9c-e2c4-41c6-9978-2b243e36be7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: "2026-04-22T19:55:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:13Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c5ec7469c3f0 namespace: kserve-ci-e2e-test resourceVersion: "25539" uid: 0a9bf773-35f5-424b-b7eb-42f0fb1360f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:03:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 5ca81ae3-fece-4202-8a80-3da8ca59a7e1 kind: Event lastTimestamp: "2026-04-22T20:03:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:03:56Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh.18a8c66fddbf03a8 namespace: kserve-ci-e2e-test resourceVersion: "29544" uid: e3a67cd1-5015-49ea-9977-7f34de6cab29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0c611-predictor-54f6b9dc8b namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: bd214041-a1b8-4147-9d12-e64d4023e7a2 kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: 'Created pod: error-404-isvc-0c611-predictor-54f6b9dc8b-hmvmh' metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:31Z" name: error-404-isvc-0c611-predictor-54f6b9dc8b.18a8c5ec566e29f9 namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: b7452455-f019-4b5a-9ab9-d0919aad12eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25104" uid: 8fcc2bc6-322b-4c25-89ea-d0a94c244c9a kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Scaled up replica set error-404-isvc-0c611-predictor-54f6b9dc8b from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:31Z" name: error-404-isvc-0c611-predictor.18a8c5ec5502a71d namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: 349df880-f272-427f-b7da-b12ab66ef087 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 4db0a0d0-e8a1-489f-9dcc-3082fda3eaee kind: Event lastTimestamp: "2026-04-22T19:55:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:16Z" name: error-404-isvc-0c611-predictor.18a8c5efd3e82d7f namespace: kserve-ci-e2e-test resourceVersion: "25570" uid: badc2b6f-a31b-40ba-981f-a98a04445d09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 4db0a0d0-e8a1-489f-9dcc-3082fda3eaee kind: Event lastTimestamp: "2026-04-22T19:55:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:16Z" name: error-404-isvc-0c611-predictor.18a8c5efd481061a namespace: kserve-ci-e2e-test resourceVersion: "25571" uid: ce3e0b0a-0517-41fa-a261-941dc418da63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:55:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 4db0a0d0-e8a1-489f-9dcc-3082fda3eaee kind: Event lastTimestamp: "2026-04-22T19:55:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:46Z" name: error-404-isvc-0c611-predictor.18a8c5fa505fae86 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: e9ebaeed-5535-4f24-93ae-213ecb7bbecb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:55:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 4db0a0d0-e8a1-489f-9dcc-3082fda3eaee kind: Event lastTimestamp: "2026-04-22T19:55:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:46Z" name: error-404-isvc-0c611-predictor.18a8c5fa50da4889 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: 11482202-3ef0-4540-a404-9269a1fafaf4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b4a1d261-23f8-4056-9b43-eb414cbba70b kind: Event lastTimestamp: "2026-04-22T19:55:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:55:23Z" name: error-404-isvc-0c611.18a8c5ec54d9da57 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 7b02b326-aff7-453a-b011-82b9e8a25329 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b4a1d261-23f8-4056-9b43-eb414cbba70b kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-0c611": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0c611": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:31Z" name: error-404-isvc-0c611.18a8c5ec553e22f0 namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7aec8912-d5d2-4656-97a2-524a1d39910c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b4a1d261-23f8-4056-9b43-eb414cbba70b kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0c611": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:31Z" name: error-404-isvc-0c611.18a8c5ec553e7f45 namespace: kserve-ci-e2e-test resourceVersion: "25111" uid: b5c7883a-8e8b-4a39-be1a-c78de216ccad reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: b4a1d261-23f8-4056-9b43-eb414cbba70b kind: Event lastTimestamp: "2026-04-22T19:55:23Z" message: InferenceService [error-404-isvc-0c611] is Ready metadata: creationTimestamp: "2026-04-22T19:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:55:23Z" name: error-404-isvc-0c611.18a8c5f8579f7083 namespace: kserve-ci-e2e-test resourceVersion: "25639" uid: 6599f622-f367-4ba4-8e3d-67982d4c27bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:33:01.379102Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c4bfe054d2db namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: 051d19de-daa2-4a94-b819-301c24160b86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c4bfe8b8a516 namespace: kserve-ci-e2e-test resourceVersion: "14420" uid: 8b2c1b08-4894-41f2-8a68-945c2485fd93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c4bfea6e249b namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 4cfa6b02-0ea4-4666-add6-aad5ffcea43f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c4bfedc85272 namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 0f037307-849c-4182-8a6e-0ce2d9e96a72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c4bfee3efd17 namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 20527f41-c9c8-4bb5-acb0-b8364af1444e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: "2026-04-22T19:33:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:43Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c4c015609caa namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: eae93fde-9db1-4294-a96e-1951275bf70c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 2d88a16b-b00a-4a5e-a612-c6aa262424ce kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z.18a8c54361a91a15 namespace: kserve-ci-e2e-test resourceVersion: "18806" uid: f9550228-9982-42ec-8937-92aa7bdf4d9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1be4a-predictor-7d9f6f6594 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: f621a66d-8c90-41a8-9998-b71a8ded3e37 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'Created pod: error-404-isvc-1be4a-predictor-7d9f6f6594-wlg8z' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor-7d9f6f6594.18a8c4bfe02a8f3a namespace: kserve-ci-e2e-test resourceVersion: "14403" uid: cbf680e5-8dba-4b17-b375-9a46eed2167e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: cc6e7f52-e01d-4a04-a0d1-9fb873334da6 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Scaled up replica set error-404-isvc-1be4a-predictor-7d9f6f6594 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a-predictor.18a8c4bfde848217 namespace: kserve-ci-e2e-test resourceVersion: "14395" uid: d074f34e-4fe1-44c6-b5f6-cceae8d744b3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 38810eec-946a-4047-a6d6-cdc3a67ba964 kind: Event lastTimestamp: "2026-04-22T19:33:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:46Z" name: error-404-isvc-1be4a-predictor.18a8c4c35d640aa7 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: ca14245c-f3f5-4363-bb33-763dee6894c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 38810eec-946a-4047-a6d6-cdc3a67ba964 kind: Event lastTimestamp: "2026-04-22T19:33:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:46Z" name: error-404-isvc-1be4a-predictor.18a8c4c35dc6d0a6 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: 299314d7-03c5-4d4a-aa9f-81ace2fade69 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:34:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 38810eec-946a-4047-a6d6-cdc3a67ba964 kind: Event lastTimestamp: "2026-04-22T19:34:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:16Z" name: error-404-isvc-1be4a-predictor.18a8c4cddbfcaba6 namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: c9d58706-3ecb-40f6-b238-9f9ce4f48723 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:34:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 38810eec-946a-4047-a6d6-cdc3a67ba964 kind: Event lastTimestamp: "2026-04-22T19:34:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:16Z" name: error-404-isvc-1be4a-predictor.18a8c4cddc8ebb17 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 448b13b6-c9f1-4b78-a3f9-4984c9f43d61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: d8101b0e-ffc4-4e5a-ac21-039a195ea58b kind: Event lastTimestamp: "2026-04-22T19:33:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:53Z" name: error-404-isvc-1be4a.18a8c4bfdebc9266 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 59d4c6a9-65be-412e-bc14-794e456ef749 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: d8101b0e-ffc4-4e5a-ac21-039a195ea58b kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-1be4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1be4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a.18a8c4bfdf2eb9fc namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: e9361151-c340-43b9-9bc9-f3fb5704f201 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: d8101b0e-ffc4-4e5a-ac21-039a195ea58b kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1be4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-1be4a.18a8c4bfdf2f2405 namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: 9c07f65d-3430-493f-b0f9-7bcad4b2f9f3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: d8101b0e-ffc4-4e5a-ac21-039a195ea58b kind: Event lastTimestamp: "2026-04-22T19:33:53Z" message: InferenceService [error-404-isvc-1be4a] is Ready metadata: creationTimestamp: "2026-04-22T19:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:53Z" name: error-404-isvc-1be4a.18a8c4cbfa32a9e3 namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: de9eff6d-753d-407e-97e8-73a2025ea3f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:33:47.528630Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-22a3d-predictor-69444449fb-mvct7 to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c4ca9f0eb329 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 4afac296-4b28-4ec9-83fa-f267825e8b1c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c4caa73a18e3 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 0ab6e7cd-3e02-4e0c-b9a6-76a1b6b505d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c4caa88f0267 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: d87d6dde-8137-49e8-a991-3833bd68ef42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c4caaf7fbfe7 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 542bb40a-1df5-4700-9c00-3ffe1aa272c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c4caaff3c97d namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: baa0536a-c6bd-47fa-a851-0eda1ca1940e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: "2026-04-22T19:34:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:29Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c4cad403a1f3 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 57b33e94-bd6f-4f15-982b-330df5fd6a49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22a3d-predictor-69444449fb-mvct7 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 891c9278-8b52-438f-9afd-f3df893b2826 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-22a3d-predictor-69444449fb-mvct7.18a8c54e1f6e4fd4 namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: 432e3e51-4ae2-4751-a743-06f7a4773622 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-22a3d-predictor-69444449fb namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 120f4dd8-88d1-4712-8042-66aa3161eccf kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'Created pod: error-404-isvc-22a3d-predictor-69444449fb-mvct7' metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor-69444449fb.18a8c4ca9e6a18ea namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: ce33fa58-e7a1-4c2b-8633-c51eca7589fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-22a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: af0e19a6-8c0b-459b-ba87-2a00f20efc12 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Scaled up replica set error-404-isvc-22a3d-predictor-69444449fb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d-predictor.18a8c4ca9c1d1017 namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 255019f5-594f-44c9-b36a-aaee104a4824 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:34:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-22a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 3d17308b-82e4-4151-9fc0-2044e00aa18a kind: Event lastTimestamp: "2026-04-22T19:34:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:47Z" name: error-404-isvc-22a3d-predictor.18a8c4ce1add7178 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 426d5106-86fa-4b7f-8f57-0581c48702ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:34:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-22a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 3d17308b-82e4-4151-9fc0-2044e00aa18a kind: Event lastTimestamp: "2026-04-22T19:34:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:47Z" name: error-404-isvc-22a3d-predictor.18a8c4ce1b4d9f01 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: cd7832dc-3ec8-437f-bf55-0b696159941f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22a3d namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 085f6056-47ba-49f6-b788-732eab52d8b8 kind: Event lastTimestamp: "2026-04-22T19:34:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:34:39Z" name: error-404-isvc-22a3d.18a8c4ca9bf6067b namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: c178e9b3-b66e-4fc1-8a09-20f508a13702 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22a3d namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 085f6056-47ba-49f6-b788-732eab52d8b8 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-22a3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-22a3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d.18a8c4ca9c593d87 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 7f73c557-e062-45f1-be05-fd5ce5564a0d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22a3d namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 085f6056-47ba-49f6-b788-732eab52d8b8 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-22a3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:47Z" name: error-404-isvc-22a3d.18a8c4ca9c5991f2 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 97ec475b-30bd-423c-a97e-b0540f654214 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22a3d namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: 085f6056-47ba-49f6-b788-732eab52d8b8 kind: Event lastTimestamp: "2026-04-22T19:34:39Z" message: InferenceService [error-404-isvc-22a3d] is Ready metadata: creationTimestamp: "2026-04-22T19:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:34:39Z" name: error-404-isvc-22a3d.18a8c4d6b7c24002 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 455b794b-58f1-487b-9c3b-10cba7ea8aba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:43:12.349531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3becf-predictor-7846f6d998-8q6cb to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c54e2101aae9 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: fcaacb7a-f64a-44a0-9c34-a8c031526477 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19319" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c54e35489e7f namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: 9db9176d-731c-4caa-955a-d096f7790d0d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c54e36716100 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: 2c2ae6f2-0385-4dc8-945a-49e4d929d073 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c54e3c1fceca namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 1529d4cf-8298-493a-8d08-110f2ef38e75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c54e3c92f033 namespace: kserve-ci-e2e-test resourceVersion: "19338" uid: 75425ec9-125e-4428-b7af-e5a7f344f721 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:43:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: "2026-04-22T19:44:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:43:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:34Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c54e533b7657 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 3584ce50-316e-44f0-ad53-0b18b169466d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 8afd4c3b-c1d1-41b7-baeb-5bcce37dff34 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-3becf-predictor-7846f6d998-8q6cb.18a8c560ce2380e7 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 727a6c19-e1ea-4234-867d-b8bb0783cff9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3becf-predictor-7846f6d998 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 7f2d9272-c725-42ae-916b-3315e3400762 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'Created pod: error-404-isvc-3becf-predictor-7846f6d998-8q6cb' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor-7846f6d998.18a8c54e207425ec namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: 27c31406-807f-40f1-b5b4-b48998b4ac28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19296" uid: ebd488ef-a5f3-4329-87d2-9f600e0c26c0 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Scaled up replica set error-404-isvc-3becf-predictor-7846f6d998 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf-predictor.18a8c54e1f0c2185 namespace: kserve-ci-e2e-test resourceVersion: "19302" uid: 48e87343-eb52-4089-b0eb-5e647a3715b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19298" uid: 2d2e8d64-0612-4bed-8ce2-9a1dadfd4d59 kind: Event lastTimestamp: "2026-04-22T19:43:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:42Z" name: error-404-isvc-3becf-predictor.18a8c5519d6b2f3b namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 5b035b73-0198-48d9-bbf6-7d23c4d43b6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19298" uid: 2d2e8d64-0612-4bed-8ce2-9a1dadfd4d59 kind: Event lastTimestamp: "2026-04-22T19:43:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:42Z" name: error-404-isvc-3becf-predictor.18a8c5519dae130e namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: d58c079f-1052-4dd3-b653-403a82cb9189 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: 2d2e8d64-0612-4bed-8ce2-9a1dadfd4d59 kind: Event lastTimestamp: "2026-04-22T19:44:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:27Z" name: error-404-isvc-3becf-predictor.18a8c5589aeef853 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: c8ccc85c-8792-461a-9889-40a5b634ca23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: 2d2e8d64-0612-4bed-8ce2-9a1dadfd4d59 kind: Event lastTimestamp: "2026-04-22T19:44:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:27Z" name: error-404-isvc-3becf-predictor.18a8c5589b4e543d namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: e4f56437-7412-4a6a-9c56-7da228b47b91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: 5d3aa738-5495-4938-9988-e5b0856c8cf5 kind: Event lastTimestamp: "2026-04-22T19:44:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:04Z" name: error-404-isvc-3becf.18a8c54e1edcad9e namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: ac85d117-dfbb-46ce-8807-1373f4508e05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: 5d3aa738-5495-4938-9988-e5b0856c8cf5 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-3becf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3becf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf.18a8c54e1f4ae18e namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 589ed2b3-6152-484a-8cad-9d1d9fb17cc8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: 5d3aa738-5495-4938-9988-e5b0856c8cf5 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3becf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:12Z" name: error-404-isvc-3becf.18a8c54e1f4b89c3 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 88473e25-0b39-45ad-bfe3-e033b98a8250 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19971" uid: 5d3aa738-5495-4938-9988-e5b0856c8cf5 kind: Event lastTimestamp: "2026-04-22T19:44:04Z" message: InferenceService [error-404-isvc-3becf] is Ready metadata: creationTimestamp: "2026-04-22T19:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:04Z" name: error-404-isvc-3becf.18a8c55a36c5b99a namespace: kserve-ci-e2e-test resourceVersion: "19972" uid: 1b28db57-ef5a-46c4-9994-5d5bb3e47c27 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:53:11.560284Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24172" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-44fda-predictor-68df868f8b-md6rk to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5d9a4c0241a namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: b6a4e2c7-95d9-4377-8e2d-1581eb703772 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24182" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5d9ac8c50a9 namespace: kserve-ci-e2e-test resourceVersion: "24190" uid: 3d1c72a0-1ca1-4542-b789-c9ff229f7af1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5d9adc7e442 namespace: kserve-ci-e2e-test resourceVersion: "24195" uid: 6883be17-650d-414a-922c-b0de405882c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5d9b34e6700 namespace: kserve-ci-e2e-test resourceVersion: "24197" uid: 99a67587-4f26-456d-816a-dda9ea4a6324 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5d9b3c07ee2 namespace: kserve-ci-e2e-test resourceVersion: "24198" uid: 9b517392-0acd-42a2-b66e-439467afc3f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:53:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: "2026-04-22T19:54:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:53:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:33Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5d9c46fb35f namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 593fedcf-2551-46df-bc5e-1d3fb6bc488e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44fda-predictor-68df868f8b-md6rk namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: a2d2887f-0262-4abc-a637-10a452c2131b kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:31Z" name: error-404-isvc-44fda-predictor-68df868f8b-md6rk.18a8c5ec4f486697 namespace: kserve-ci-e2e-test resourceVersion: "25066" uid: 03f29ece-12f6-4391-9dbf-1de291e1d6dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-44fda-predictor-68df868f8b namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: 6bb6f930-1163-49ac-811f-c5385d03131f kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: 'Created pod: error-404-isvc-44fda-predictor-68df868f8b-md6rk' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor-68df868f8b.18a8c5d9a458e179 namespace: kserve-ci-e2e-test resourceVersion: "24173" uid: eeea5d92-4bb5-4d46-b2da-455c5f2ef828 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24158" uid: 3295e5e7-c87b-4d55-949f-5582c719ffac kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Scaled up replica set error-404-isvc-44fda-predictor-68df868f8b from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda-predictor.18a8c5d9a23786a9 namespace: kserve-ci-e2e-test resourceVersion: "24166" uid: b28b202f-f6b9-40eb-a778-d60fd42c9fe1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: 6689e889-939f-4dd7-af2d-8172b6278eb6 kind: Event lastTimestamp: "2026-04-22T19:53:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:26Z" name: error-404-isvc-44fda-predictor.18a8c5dd20aa86ff namespace: kserve-ci-e2e-test resourceVersion: "24325" uid: 65477fff-f626-4251-9a3a-8461ae6c89ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: 6689e889-939f-4dd7-af2d-8172b6278eb6 kind: Event lastTimestamp: "2026-04-22T19:53:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:26Z" name: error-404-isvc-44fda-predictor.18a8c5dd21158615 namespace: kserve-ci-e2e-test resourceVersion: "24327" uid: 20e44409-ba55-4df9-a394-3aa7407d3cd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:53:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: 6689e889-939f-4dd7-af2d-8172b6278eb6 kind: Event lastTimestamp: "2026-04-22T19:54:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:26Z" name: error-404-isvc-44fda-predictor.18a8c5e09fab8734 namespace: kserve-ci-e2e-test resourceVersion: "24971" uid: 48d817a7-3340-48ca-8603-346260ea3539 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:53:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: 6689e889-939f-4dd7-af2d-8172b6278eb6 kind: Event lastTimestamp: "2026-04-22T19:54:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:26Z" name: error-404-isvc-44fda-predictor.18a8c5e0a010c8a7 namespace: kserve-ci-e2e-test resourceVersion: "24972" uid: 6b806654-c9ac-4c0c-92b6-e861b28b1d1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44fda namespace: kserve-ci-e2e-test resourceVersion: "24141" uid: 83203d92-f4c1-452f-bc2a-764559cc6709 kind: Event lastTimestamp: "2026-04-22T19:54:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:03Z" name: error-404-isvc-44fda.18a8c5d9a212ff0a namespace: kserve-ci-e2e-test resourceVersion: "24746" uid: 5627209f-2b74-4f3b-8ce1-62c192925944 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44fda namespace: kserve-ci-e2e-test resourceVersion: "24141" uid: 83203d92-f4c1-452f-bc2a-764559cc6709 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-44fda": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-44fda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda.18a8c5d9a297bebc namespace: kserve-ci-e2e-test resourceVersion: "24168" uid: fcc90d01-3d94-4dad-9cf7-5576124616cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44fda namespace: kserve-ci-e2e-test resourceVersion: "24141" uid: 83203d92-f4c1-452f-bc2a-764559cc6709 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-44fda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-44fda.18a8c5d9a2981e10 namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: e7eef7a3-65b7-4ca5-b6e3-9811366ad804 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44fda namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 83203d92-f4c1-452f-bc2a-764559cc6709 kind: Event lastTimestamp: "2026-04-22T19:54:03Z" message: InferenceService [error-404-isvc-44fda] is Ready metadata: creationTimestamp: "2026-04-22T19:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:03Z" name: error-404-isvc-44fda.18a8c5e5a857dbec namespace: kserve-ci-e2e-test resourceVersion: "24745" uid: dfa46f79-ec45-49f2-b691-72afced3834a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:43:46.603744Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-62743-predictor-54cbd885cb-cz89z to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c5561ab7733f namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 46713fb5-8ea2-4cf3-bfbe-1d6b03e9c75b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c55622c7d252 namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: 147c1360-1581-4a8a-b3da-a99567434feb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c55624165ab1 namespace: kserve-ci-e2e-test resourceVersion: "19794" uid: 0dcb7483-7809-4224-a389-763b0b90df52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c5562745a87c namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: afbada4b-85c2-40b8-8f24-29dc9aa26b5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c55627bdc8c3 namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: d19b5dcb-9229-4216-8fa9-b201b2597f75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:43:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: "2026-04-22T19:44:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:28Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c55644bce73e namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 6c22f5aa-0d66-448d-94c1-03eda49736f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62743-predictor-54cbd885cb-cz89z namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: d16a84c8-15e0-40f5-8a21-70ae10ee4b53 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: error-404-isvc-62743-predictor-54cbd885cb-cz89z.18a8c5d99f9129bd namespace: kserve-ci-e2e-test resourceVersion: "24144" uid: 98f7038b-4643-4f16-918c-7cc5495043f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-62743-predictor-54cbd885cb namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: be224830-df71-406a-9205-6e5b3bd8d3e7 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: 'Created pod: error-404-isvc-62743-predictor-54cbd885cb-cz89z' metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor-54cbd885cb.18a8c5561a4e9161 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: d5eeb3ac-48f2-4c14-a1e2-68323eac3b0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: 1c3e3762-2443-41e8-bccb-75bf1db7055f kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Scaled up replica set error-404-isvc-62743-predictor-54cbd885cb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743-predictor.18a8c55618f8b790 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 5de6931a-cf5f-4ed1-a3ac-7a9ca030425f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 56abbde1-6546-4d34-9f07-152604980bc5 kind: Event lastTimestamp: "2026-04-22T19:44:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:01Z" name: error-404-isvc-62743-predictor.18a8c55997616bf8 namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: 0835ad75-8443-402c-894f-adfecfcbb437 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 56abbde1-6546-4d34-9f07-152604980bc5 kind: Event lastTimestamp: "2026-04-22T19:44:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:01Z" name: error-404-isvc-62743-predictor.18a8c55997c65cab namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: a0293c78-f610-4761-9758-ca2907aa0d22 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: 56abbde1-6546-4d34-9f07-152604980bc5 kind: Event lastTimestamp: "2026-04-22T19:45:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:16Z" name: error-404-isvc-62743-predictor.18a8c55d1650cbb6 namespace: kserve-ci-e2e-test resourceVersion: "20778" uid: 5a4f7db7-a876-43fd-8baf-e687fe979185 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: 56abbde1-6546-4d34-9f07-152604980bc5 kind: Event lastTimestamp: "2026-04-22T19:45:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:16Z" name: error-404-isvc-62743-predictor.18a8c55d16b6217b namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: b8fda778-b9d9-42ae-af78-b323c1286978 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62743 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: c6776c5d-e236-470e-9013-53a9daaf77b4 kind: Event lastTimestamp: "2026-04-22T19:44:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:38Z" name: error-404-isvc-62743.18a8c55618d0a8af namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 12771f5e-f752-49cc-93cb-922b4798167b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62743 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: c6776c5d-e236-470e-9013-53a9daaf77b4 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-62743": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-62743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743.18a8c556191ab2ef namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: bb049861-7d64-4adc-abeb-65c0bfa3ae11 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62743 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: c6776c5d-e236-470e-9013-53a9daaf77b4 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-62743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:46Z" name: error-404-isvc-62743.18a8c556191afe56 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 0ddfd28a-df1c-4774-a8fa-f32ebf030e8f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62743 namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: c6776c5d-e236-470e-9013-53a9daaf77b4 kind: Event lastTimestamp: "2026-04-22T19:44:38Z" message: InferenceService [error-404-isvc-62743] is Ready metadata: creationTimestamp: "2026-04-22T19:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:38Z" name: error-404-isvc-62743.18a8c5622813187c namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: c12c6d76-1390-41d4-923b-36d975cb7b7c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:53:57.500778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24639" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-65828-predictor-7fc45d4b74-sspp4 to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c5e457047765 namespace: kserve-ci-e2e-test resourceVersion: "24643" uid: 9199cda3-311f-4fc4-8fcc-c68c496e91be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24649" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c5e45f82ec74 namespace: kserve-ci-e2e-test resourceVersion: "24658" uid: f250522c-c74e-4943-8fdd-1bdfcd20dc2f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c5e460c422c3 namespace: kserve-ci-e2e-test resourceVersion: "24661" uid: 154462bc-35fb-42ea-8f26-0dc6938bb180 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c5e4692c2415 namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: 66f8d0dc-d9ed-4f85-b326-39b6df6fae4f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c5e469a4f0c8 namespace: kserve-ci-e2e-test resourceVersion: "24666" uid: 4ecf06e6-89b3-42b6-92e9-6027b2efb07d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:53:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: "2026-04-22T20:11:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:29Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c5e48355b9c4 namespace: kserve-ci-e2e-test resourceVersion: "32719" uid: 14fa29f9-2b53-4b20-8d9a-787d20d3e335 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: 37726911-fdc0-4a6e-86e4-4aba23aa46d1 kind: Event lastTimestamp: "2026-04-22T20:11:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:26Z" name: error-404-isvc-65828-predictor-7fc45d4b74-sspp4.18a8c6d8aed6d8fb namespace: kserve-ci-e2e-test resourceVersion: "32668" uid: 3f6be513-efe4-430a-924c-156a5be674b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-65828-predictor-7fc45d4b74 namespace: kserve-ci-e2e-test resourceVersion: "24631" uid: be8caaea-9b8c-4c6e-b4e5-274d1c9ac5e7 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: 'Created pod: error-404-isvc-65828-predictor-7fc45d4b74-sspp4' metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor-7fc45d4b74.18a8c5e456831ccf namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: a94c8af8-6007-41fd-8f3a-1a58c2c167bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24629" uid: 7b798ee7-e46b-4584-a227-18cab01ab617 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Scaled up replica set error-404-isvc-65828-predictor-7fc45d4b74 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828-predictor.18a8c5e4555ec799 namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: c4a16a5c-b4a4-470e-b77b-a06c4b8da4bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:54:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24630" uid: e4c26617-2c7a-40ef-abae-edd106736137 kind: Event lastTimestamp: "2026-04-22T19:54:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:57Z" name: error-404-isvc-65828-predictor.18a8c5e7d3cda0af namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: edd15e76-efa6-41d1-8766-951478ad62ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:54:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24630" uid: e4c26617-2c7a-40ef-abae-edd106736137 kind: Event lastTimestamp: "2026-04-22T19:54:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:57Z" name: error-404-isvc-65828-predictor.18a8c5e7d43773a8 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: a1b48c39-e211-459a-b0be-20e27ad90264 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:55:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24824" uid: e4c26617-2c7a-40ef-abae-edd106736137 kind: Event lastTimestamp: "2026-04-22T19:55:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:27Z" name: error-404-isvc-65828-predictor.18a8c5f5ced37c35 namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: ae99df1a-894a-4cdd-ba99-bee9083454d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:55:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24824" uid: e4c26617-2c7a-40ef-abae-edd106736137 kind: Event lastTimestamp: "2026-04-22T19:55:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:27Z" name: error-404-isvc-65828-predictor.18a8c5f5cf176d07 namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: e6c3a3bb-713c-409d-a1fc-09ed742d5e67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65828 namespace: kserve-ci-e2e-test resourceVersion: "24611" uid: f5e8f8db-877b-4e52-8678-714f127ff8d7 kind: Event lastTimestamp: "2026-04-22T19:54:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:49Z" name: error-404-isvc-65828.18a8c5e45533e457 namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: de32a1f5-22f2-4e77-aa00-4c99a8188adc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65828 namespace: kserve-ci-e2e-test resourceVersion: "24611" uid: f5e8f8db-877b-4e52-8678-714f127ff8d7 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-65828": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-65828": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828.18a8c5e45580bc8d namespace: kserve-ci-e2e-test resourceVersion: "24635" uid: fe671a1a-dd2e-479c-b086-db4d9743290c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65828 namespace: kserve-ci-e2e-test resourceVersion: "24611" uid: f5e8f8db-877b-4e52-8678-714f127ff8d7 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-65828": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-65828.18a8c5e455810619 namespace: kserve-ci-e2e-test resourceVersion: "24636" uid: 9efb328e-7efe-4a29-a79c-eb0a7452f3bf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65828 namespace: kserve-ci-e2e-test resourceVersion: "25300" uid: f5e8f8db-877b-4e52-8678-714f127ff8d7 kind: Event lastTimestamp: "2026-04-22T19:54:49Z" message: InferenceService [error-404-isvc-65828] is Ready metadata: creationTimestamp: "2026-04-22T19:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:49Z" name: error-404-isvc-65828.18a8c5f066d1a522 namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: e39de4f5-9552-4dd0-97a4-4e842be5d24b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:32.593668Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c560cfee022a namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: 7525d4ae-f034-4553-8976-37aaf66043bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c560d87ac230 namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 308825c3-1c8a-4164-9883-70809f460f3c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c560d9a9fc8e namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: 9491e2db-b79f-4e70-982e-58016fb773dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c560e276cfd9 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: d61c7041-2aa8-4ad9-a464-f61b0da68e53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c560e2f05831 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 19e7faab-fd84-4875-bcd7-8dce63332685 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: "2026-04-22T19:45:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:14Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c561034f80fb namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: c4c7f785-6add-417b-9130-25d4530ea9e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 611955a8-9b65-46d1-a592-5a176628ebed kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv.18a8c5e4537c87dc namespace: kserve-ci-e2e-test resourceVersion: "24620" uid: 0a5cca9b-cd89-4184-8ebd-f24dd10744b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7d5c8-predictor-595fbbfc94 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: ed1cb902-ca3f-4e99-bc08-c76e7fff4d13 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'Created pod: error-404-isvc-7d5c8-predictor-595fbbfc94-q5ghv' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor-595fbbfc94.18a8c560cf821c7b namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: 50b31316-2d54-448e-96af-7e7ec62237ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 747e9db6-2209-43f7-93c4-776679b3c1ab kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Scaled up replica set error-404-isvc-7d5c8-predictor-595fbbfc94 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8-predictor.18a8c560ce13971d namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 01d0a529-5fe6-4f22-af45-3fea023bce3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 7dd4a32d-3fae-440f-ab72-167a025a39f2 kind: Event lastTimestamp: "2026-04-22T19:44:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:47Z" name: error-404-isvc-7d5c8-predictor.18a8c5644cb1b1cb namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: e7b9ef4b-c186-470f-b712-62856b75afea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 7dd4a32d-3fae-440f-ab72-167a025a39f2 kind: Event lastTimestamp: "2026-04-22T19:44:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:47Z" name: error-404-isvc-7d5c8-predictor.18a8c5644d137efe namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 2e70f672-2a00-4993-afcb-c0532329aa7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:45:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 7dd4a32d-3fae-440f-ab72-167a025a39f2 kind: Event lastTimestamp: "2026-04-22T19:45:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:32Z" name: error-404-isvc-7d5c8-predictor.18a8c567cba9bee7 namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: 5565dfd9-11ea-40f1-9818-e1ab2a398369 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:45:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 7dd4a32d-3fae-440f-ab72-167a025a39f2 kind: Event lastTimestamp: "2026-04-22T19:45:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:32Z" name: error-404-isvc-7d5c8-predictor.18a8c567cc0c9e7d namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 9c3695a2-db88-476a-802c-89729bde4296 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 397c5e4f-46a7-43c6-a9b9-fdd015b3cdbe kind: Event lastTimestamp: "2026-04-22T19:45:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:45:24Z" name: error-404-isvc-7d5c8.18a8c560ce1b4fae namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 38ff855f-abbc-4db2-9b60-0dbad2c1967f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 397c5e4f-46a7-43c6-a9b9-fdd015b3cdbe kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-7d5c8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7d5c8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8.18a8c560cebf5d7f namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: f5f77479-0562-4785-bc16-0cb8b6997db3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 397c5e4f-46a7-43c6-a9b9-fdd015b3cdbe kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7d5c8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:32Z" name: error-404-isvc-7d5c8.18a8c560cebfa800 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: e9f74817-b7c3-47cf-9fb6-4cd564128df4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 397c5e4f-46a7-43c6-a9b9-fdd015b3cdbe kind: Event lastTimestamp: "2026-04-22T19:45:24Z" message: InferenceService [error-404-isvc-7d5c8] is Ready metadata: creationTimestamp: "2026-04-22T19:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:45:24Z" name: error-404-isvc-7d5c8.18a8c56ce6ad088a namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 421af609-8704-4489-8d26-38fd73c08407 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:31:27.212406Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13332" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k to ip-10-0-129-145.ec2.internal metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:31:27Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4a9f38f06c2 namespace: kserve-ci-e2e-test resourceVersion: "13339" uid: 3516eb29-82fc-46d7-abaf-67fe9463308f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13349" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:31:27Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4a9fe700469 namespace: kserve-ci-e2e-test resourceVersion: "13382" uid: c94bd254-e928-4f9b-860a-c5ac05865b83 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:27Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4a9ffd4ebe4 namespace: kserve-ci-e2e-test resourceVersion: "13384" uid: 6ca005bf-f653-46f8-8367-ba8a93b5dba7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:31:41Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1256" in 14.068s (14.068s including waiting). Image size: 1334191706 bytes.' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:41Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4ad46681e78 namespace: kserve-ci-e2e-test resourceVersion: "13573" uid: 9bd92b41-3bf6-43ec-a8f2-941afa692f07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:31:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:41Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4ad4d6e350b namespace: kserve-ci-e2e-test resourceVersion: "13574" uid: d04c9d31-522b-4903-afe8-4411343efbd5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:31:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:41Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4ad4de5d61c namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: 5529ae78-286c-4997-bdcc-dcf51f0c48af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:33:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:03Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4ad660274b9 namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 0a9a1b69-e248-4031-8229-f26998376ee9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: 4c8c27be-2412-4c38-b418-a6b4f24b78d9 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k.18a8c4bfdf00d5af namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: b20c5d92-e79f-4ad8-b626-06d6e59dfa6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-145.ec2.internal source: component: kubelet host: ip-10-0-129-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2ca6-predictor-6f7678757b namespace: kserve-ci-e2e-test resourceVersion: "13325" uid: dbe3de1f-cb08-4d05-abab-334cf35c1dd5 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: 'Created pod: error-404-isvc-b2ca6-predictor-6f7678757b-q5b8k' metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: error-404-isvc-b2ca6-predictor-6f7678757b.18a8c4a9f308eff8 namespace: kserve-ci-e2e-test resourceVersion: "13334" uid: 2a008909-6f9c-4e2e-b8a8-13590bcaca9a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13321" uid: 0d2ea055-ebb7-4007-a4e3-d807585b2bbe kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Scaled up replica set error-404-isvc-b2ca6-predictor-6f7678757b from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: error-404-isvc-b2ca6-predictor.18a8c4a9f0a92451 namespace: kserve-ci-e2e-test resourceVersion: "13326" uid: 025878f9-08c6-42d6-8176-b7b51b4ab178 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13324" uid: 0fdf7bb8-8a01-48af-882b-f78d8908302a kind: Event lastTimestamp: "2026-04-22T19:32:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:12Z" name: error-404-isvc-b2ca6-predictor.18a8c4ad6f3f00ce namespace: kserve-ci-e2e-test resourceVersion: "13849" uid: d5e0a846-3438-4470-bf4f-b86b63b41a4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13324" uid: 0fdf7bb8-8a01-48af-882b-f78d8908302a kind: Event lastTimestamp: "2026-04-22T19:32:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:12Z" name: error-404-isvc-b2ca6-predictor.18a8c4ad6fa5197c namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: cbae22c4-15f8-41ad-8aec-e9c1127ce692 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:32:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: 0fdf7bb8-8a01-48af-882b-f78d8908302a kind: Event lastTimestamp: "2026-04-22T19:32:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:57Z" name: error-404-isvc-b2ca6-predictor.18a8c4b7ebbadb8b namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 48be32ee-2a6f-4f39-a41a-a38a6d532054 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:32:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: 0fdf7bb8-8a01-48af-882b-f78d8908302a kind: Event lastTimestamp: "2026-04-22T19:32:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:57Z" name: error-404-isvc-b2ca6-predictor.18a8c4b7ec1d555e namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: ec42fb22-3277-4a73-9efb-823974754d5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "13307" uid: c6c0ddd4-f863-424e-8265-1b479d8ece3f kind: Event lastTimestamp: "2026-04-22T19:32:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:33Z" name: error-404-isvc-b2ca6.18a8c4a9f0ae6a6a namespace: kserve-ci-e2e-test resourceVersion: "14044" uid: b30b58b6-f8c0-49a8-b170-00e14df95b81 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: c6c0ddd4-f863-424e-8265-1b479d8ece3f kind: Event lastTimestamp: "2026-04-22T19:32:33Z" message: InferenceService [error-404-isvc-b2ca6] is Ready metadata: creationTimestamp: "2026-04-22T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:33Z" name: error-404-isvc-b2ca6.18a8c4b949bce76f namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: 272f7643-1c69-47da-a0bc-cc4b69e19a52 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:31:27.135111Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13308" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 to ip-10-0-129-175.ec2.internal metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4a9eef3927b namespace: kserve-ci-e2e-test resourceVersion: "13314" uid: ef76b6e1-d17f-4573-8dfb-dbcab99fda0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13322" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4aa082f2998 namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: dabd6b1e-ccff-4ae5-8e74-dc061f13b471 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4aa098cf441 namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: a3413dad-53aa-44f8-9787-9f838edce4e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:31:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" in 4.226s (4.226s including waiting). Image size: 299660181 bytes.' metadata: creationTimestamp: "2026-04-22T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:31Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4ab0576679e namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: ca3856c3-8619-4e01-a1d9-2a334e2e3a3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:31:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:31Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4ab09b85db5 namespace: kserve-ci-e2e-test resourceVersion: "13462" uid: 253462cb-4b4d-4003-b126-c65084976aa5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:31:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:31Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4ab0a2ec31b namespace: kserve-ci-e2e-test resourceVersion: "13465" uid: d935e00d-4752-44d7-aa29-1e246ccd521b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:31:36Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" metadata: creationTimestamp: "2026-04-22T19:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:36Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4ac12dcdd69 namespace: kserve-ci-e2e-test resourceVersion: "13520" uid: 4042a0c8-c375-4d06-9a31-b94b510c87ae reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:32:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" in 23.993s (23.993s including waiting). Image size: 1565185957 bytes.' metadata: creationTimestamp: "2026-04-22T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:00Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4b1a9029b6e namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: 137fd087-a9a6-40ab-8eb4-26380f49a3ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:32:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:00Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4b1ad96844a namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: 0c6e89e0-da33-4d1e-93b3-9c343c8031a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:32:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:00Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4b1aee1cc5d namespace: kserve-ci-e2e-test resourceVersion: "13756" uid: 1565488a-3044-44f5-a2da-7ecfb436302f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T19:32:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:33:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:02Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4b1eaacafef namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 6898b65b-3cb6-4176-b941-a87267ee2968 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 2b502dc7-70d7-4339-a699-6072ff2e605b kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6.18a8c4ca99210ff4 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 7ced126f-55ad-4040-ae69-b56fead06b58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7988f94697 namespace: kserve-ci-e2e-test resourceVersion: "13299" uid: cb29b5cb-8ce4-483f-8505-10b16cdf2887 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7988f94697-hbhs6' metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-1-predictor-7988f94697.18a8c4a9ee7bd828 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 261df030-e360-414b-aa54-5c274e66c466 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ea6d1c77-9b73-4fa7-af30-43b6dc348609 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7988f94697 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-1-predictor.18a8c4a9ec195dba namespace: kserve-ci-e2e-test resourceVersion: "13302" uid: d3bda0e9-eaf7-4b91-ad5d-7d2488b3c7aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13298" uid: 9c338d88-ed9a-4e80-9393-2eeba0ab395a kind: Event lastTimestamp: "2026-04-22T19:32:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:12Z" name: isvc-sklearn-graph-1-predictor.18a8c4ad6a9176de namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: a7550502-1fd1-4930-8f96-2f723b8e444d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13298" uid: 9c338d88-ed9a-4e80-9393-2eeba0ab395a kind: Event lastTimestamp: "2026-04-22T19:32:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:12Z" name: isvc-sklearn-graph-1-predictor.18a8c4ad6af7fe60 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: fbfb292f-4bdc-4477-8cba-d9104b296b9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:32:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: 9c338d88-ed9a-4e80-9393-2eeba0ab395a kind: Event lastTimestamp: "2026-04-22T19:33:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:27Z" name: isvc-sklearn-graph-1-predictor.18a8c4b7e74427b3 namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 76b76cf6-8682-4112-832e-a20d7146e971 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:32:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: 9c338d88-ed9a-4e80-9393-2eeba0ab395a kind: Event lastTimestamp: "2026-04-22T19:33:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:27Z" name: isvc-sklearn-graph-1-predictor.18a8c4b7e7a2de42 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 9da97ba6-ed97-472c-afa7-0ce377a3e50a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: 4afef97f-c041-4ce2-ad7b-d87866d24ffb kind: Event lastTimestamp: "2026-04-22T19:33:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:12Z" name: isvc-sklearn-graph-1.18a8c4a9ebf7d999 namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: fe5dd5e3-2a42-4b2a-b5af-d66918ae4202 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 4afef97f-c041-4ce2-ad7b-d87866d24ffb kind: Event lastTimestamp: "2026-04-22T19:33:12Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-22T19:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:12Z" name: isvc-sklearn-graph-1.18a8c4c2767420e5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 581bbb8d-dc97-4189-a59a-0dd6bb0a2795 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:31:27.498189Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4aa0497b4fd namespace: kserve-ci-e2e-test resourceVersion: "13393" uid: 8c39a23f-17b9-422f-8023-bb87da302272 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13397" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4aa21b1b597 namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: bb372062-e128-460e-8a70-1189ac2299ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:28Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" metadata: creationTimestamp: "2026-04-22T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:28Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4aa232d90b0 namespace: kserve-ci-e2e-test resourceVersion: "13417" uid: 880da06e-bdf6-4ac7-a0db-cc96f98fde6c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" in 6.051s (6.051s including waiting). Image size: 299660181 bytes.' metadata: creationTimestamp: "2026-04-22T19:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:34Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ab8be05e99 namespace: kserve-ci-e2e-test resourceVersion: "13495" uid: 91510ecd-25e3-4848-92f7-c4894e4448c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:34Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ab8f580164 namespace: kserve-ci-e2e-test resourceVersion: "13496" uid: a2701025-f321-4d79-b107-543d0367d5bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:34Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ab8fc52f1e namespace: kserve-ci-e2e-test resourceVersion: "13497" uid: 07b83fa6-c677-4b35-bd0a-f2547d5a0135 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:39Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" metadata: creationTimestamp: "2026-04-22T19:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:39Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ace9593a21 namespace: kserve-ci-e2e-test resourceVersion: "13554" uid: aa380f98-7bf2-4f8e-b9b2-e56cf0d06a4a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:46Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" in 6.701s (6.701s including waiting). Image size: 1565185957 bytes.' metadata: creationTimestamp: "2026-04-22T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:46Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ae78cc7f3e namespace: kserve-ci-e2e-test resourceVersion: "13631" uid: 95bdb2c7-4319-47fa-bc7b-cd43f1eb32b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:46Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ae7c222966 namespace: kserve-ci-e2e-test resourceVersion: "13633" uid: 2dce1822-0356-4dd9-808b-3bc3120b17f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:31:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:46Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ae7c921500 namespace: kserve-ci-e2e-test resourceVersion: "13634" uid: a159bbd6-80c8-4a63-9717-bca66b89883b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ae8c2a62bd namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 742cf9fd-f5e0-462a-9325-832a8fe44805 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: ca101244-9dd1-4827-887a-d36038109281 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd.18a8c4ca9686141b namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 48af1aa8-4df8-40e8-887c-5a84d9fe7e90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5dfcf87bd5 namespace: kserve-ci-e2e-test resourceVersion: "13365" uid: dd6d13fc-7b91-4a79-856e-12379e304b2f kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5dfcf87bd5-vw8nd' metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-2-predictor-5dfcf87bd5.18a8c4aa04320e91 namespace: kserve-ci-e2e-test resourceVersion: "13390" uid: cbe1974b-4726-4a7b-a5d3-b9a36dd16b8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13363" uid: c6b8de4e-c719-41c3-99a0-864760785450 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5dfcf87bd5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: isvc-sklearn-graph-2-predictor.18a8c4a9f99a09bc namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: a7564229-83a7-448d-963a-0c8e4a450851 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13364" uid: 414835c3-534f-4d8d-bd57-fda0f3c16833 kind: Event lastTimestamp: "2026-04-22T19:31:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:57Z" name: isvc-sklearn-graph-2-predictor.18a8c4ad77f8248e namespace: kserve-ci-e2e-test resourceVersion: "13724" uid: e85caf29-000f-41b8-8298-afba73e8f4fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13364" uid: 414835c3-534f-4d8d-bd57-fda0f3c16833 kind: Event lastTimestamp: "2026-04-22T19:31:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:57Z" name: isvc-sklearn-graph-2-predictor.18a8c4ad78594220 namespace: kserve-ci-e2e-test resourceVersion: "13725" uid: 5ae39316-49b7-4bfd-8aaa-69c6476ebe3d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:32:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13602" uid: 414835c3-534f-4d8d-bd57-fda0f3c16833 kind: Event lastTimestamp: "2026-04-22T19:33:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:12Z" name: isvc-sklearn-graph-2-predictor.18a8c4b475b6e589 namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: ac1b7af7-f35d-48fb-bab2-8291bea23a92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:32:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13602" uid: 414835c3-534f-4d8d-bd57-fda0f3c16833 kind: Event lastTimestamp: "2026-04-22T19:33:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:12Z" name: isvc-sklearn-graph-2-predictor.18a8c4b47619674a namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: b6b5cba7-5475-41b3-af55-627fe30395cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13356" uid: 40aef512-d260-4b01-af6d-a2877c4fa1ce kind: Event lastTimestamp: "2026-04-22T19:32:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:58Z" name: isvc-sklearn-graph-2.18a8c4a9f96d6f4a namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 383b374a-5756-4a14-af02-62319887917f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 40aef512-d260-4b01-af6d-a2877c4fa1ce kind: Event lastTimestamp: "2026-04-22T19:32:58Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-22T19:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:58Z" name: isvc-sklearn-graph-2.18a8c4bf17bff575 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: 017299c6-09d6-47a1-a456-a84c0a51f1b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:31:27.324495Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-xx2ph to ip-10-0-129-175.ec2.internal metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:31:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4a9fa3d57b5 namespace: kserve-ci-e2e-test resourceVersion: "13376" uid: 2577eb26-c4ef-4108-a669-6cafa182f6f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13381" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:31:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4aa130e7ee7 namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: 976063b4-65f1-4cb0-9b24-2d37485c3cc9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4aa1455715d namespace: kserve-ci-e2e-test resourceVersion: "13408" uid: c81384af-21bc-4671-a01a-3a61bdfa4e0d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:31:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" in 4.055s (4.055s including waiting). Image size: 299660181 bytes.' metadata: creationTimestamp: "2026-04-22T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4ab06131d11 namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 505c0439-3854-4b9b-ad57-dba542984bbb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:31:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4ab09d7f709 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: 69c897c8-97a4-4195-b7e7-eeaf72ab3293 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:31:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4ab0a48e77e namespace: kserve-ci-e2e-test resourceVersion: "13467" uid: 2833ed60-fd61-46d7-a071-0ef5d27db644 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:31:36Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-22T19:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:36Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4ac12f1f135 namespace: kserve-ci-e2e-test resourceVersion: "13521" uid: bd7f932d-9121-4c18-a269-a81688122261 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:32:00Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.303s (24.303s including waiting). Image size: 1306326779 bytes.' metadata: creationTimestamp: "2026-04-22T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:00Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4b1bb910d82 namespace: kserve-ci-e2e-test resourceVersion: "13757" uid: fe424991-f1a9-477d-a092-7d86122d46c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:32:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:00Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4b1c1feca97 namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: bac96a79-e575-41a6-8582-d785827fe630 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:32:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:00Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4b1c36c4c25 namespace: kserve-ci-e2e-test resourceVersion: "13760" uid: 9aafdf42-e7f2-4054-ac88-4c97c0f7e5ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:32:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:32:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:52Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4b1eaac1dac namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 5f6500c1-e20c-4b69-92ab-b05def19c66d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f2d66057-78ce-40fc-b24b-8a7fab95a623 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: isvc-xgboost-graph-predictor-784cb989b8-xx2ph.18a8c4ca9ff6d8c6 namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: cd981669-1377-42e0-97ca-fe038714eac8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-175.ec2.internal source: component: kubelet host: ip-10-0-129-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "13348" uid: 93e93d73-0b58-4beb-93a5-331863d7507a kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-xx2ph' metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a8c4a9f9c80665 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: a5f36460-492f-425f-9496-7f35c7f8b2a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13344" uid: 3b459825-439f-4124-a5bf-314bffe6c4c7 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:27Z" name: isvc-xgboost-graph-predictor.18a8c4a9f4efa216 namespace: kserve-ci-e2e-test resourceVersion: "13351" uid: c5b8a94a-5e28-40a5-b326-9f9dea713eac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13346" uid: 8a911c7e-3ba5-4cfd-9998-efdc109b0a15 kind: Event lastTimestamp: "2026-04-22T19:32:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:42Z" name: isvc-xgboost-graph-predictor.18a8c4ad734e49e2 namespace: kserve-ci-e2e-test resourceVersion: "14113" uid: 30806a9a-c367-47cb-bd60-3350cc2a1309 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13346" uid: 8a911c7e-3ba5-4cfd-9998-efdc109b0a15 kind: Event lastTimestamp: "2026-04-22T19:32:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:42Z" name: isvc-xgboost-graph-predictor.18a8c4ad73a91d92 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: ded24b16-1d57-4b25-9c0f-682ff68084f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:32:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: 8a911c7e-3ba5-4cfd-9998-efdc109b0a15 kind: Event lastTimestamp: "2026-04-22T19:33:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:27Z" name: isvc-xgboost-graph-predictor.18a8c4beecd81972 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 515ad0ad-9230-4378-8894-ab0ffbee619c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:32:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: 8a911c7e-3ba5-4cfd-9998-efdc109b0a15 kind: Event lastTimestamp: "2026-04-22T19:33:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:27Z" name: isvc-xgboost-graph-predictor.18a8c4beed375ce6 namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: aa0cfd7e-b597-4c1d-8389-f22b78957c57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13331" uid: 298cda17-7c50-44ee-9a60-17292bc78149 kind: Event lastTimestamp: "2026-04-22T19:33:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:02Z" name: isvc-xgboost-graph.18a8c4a9f4cbd9f1 namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: f9d540e9-f937-4858-ac0e-25b38a591305 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 298cda17-7c50-44ee-9a60-17292bc78149 kind: Event lastTimestamp: "2026-04-22T19:33:02Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-22T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:02Z" name: isvc-xgboost-graph.18a8c4c0226ee8cf namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 2a3741ea-2775-4d73-ba58-9e00a798e0ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:33:36.993121Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-766d6fc4c8-mvrn7 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:33:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:33:36Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4c82b17908e namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: fdf66dba-d9a5-4cda-9ef7-384b26b15139 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:33:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:37Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4c833f60e4d namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 78c17655-a069-47ee-984a-9163d4103f0b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:33:38Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:33:38Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4c86769744e namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: b86576cb-e165-44b0-a6f4-9180656756ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:33:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:38Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4c8689ac533 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 18d72e27-2c42-48ea-a33b-da4b2d966940 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:33:38Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-22T19:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:38Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4c86b92f2a4 namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: 234809c0-1ec0-4bf3-92ad-4cc263b04631 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:33:38Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-22T19:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:38Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4c86becbb3d namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: d21334f3-77ec-4934-9ae6-fcb08df48284 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4ca89328353 namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 8d975733-eb0b-4d50-80b8-451cd08e2e90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-766d6fc4c8-mvrn7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 10cd2779-fa41-433d-9d78-f64350441e58 kind: Event lastTimestamp: "2026-04-22T19:34:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:14Z" name: model-chainer-766d6fc4c8-mvrn7.18a8c4cb08d4d713 namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 561709d7-00fd-4706-a1ee-a673dd4a621a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-766d6fc4c8 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 11cabe27-b2be-4aa5-ba60-3251df8bc0cf kind: Event lastTimestamp: "2026-04-22T19:33:36Z" message: 'Created pod: model-chainer-766d6fc4c8-mvrn7' metadata: creationTimestamp: "2026-04-22T19:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:36Z" name: model-chainer-766d6fc4c8.18a8c4c82abbebf6 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: ea9f595f-706b-4047-8533-6071e56443de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: cebad9b4-4091-4b49-b5eb-d77175f0e124 kind: Event lastTimestamp: "2026-04-22T19:33:36Z" message: Scaled up replica set model-chainer-766d6fc4c8 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:36Z" name: model-chainer.18a8c4c829e4a6c7 namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 182f3489-930e-475d-8fd1-eca4aa092c4f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: b69df91a-ebe6-4611-b1eb-80e5f9503a06 kind: Event lastTimestamp: "2026-04-22T19:33:37Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-22T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:37Z" name: model-chainer.18a8c4c82ef4fc3d namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 48c4013d-acb7-4917-92bc-c53da93e9361 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:34:57.366666Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-22a3d-5498b7b956-qwg8j to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:34:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:34:57Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c4dae1ba86db namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 4ad5da5d-2005-4db6-8581-332d133877b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:34:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-22a3d-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:57Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c4daecdc2e53 namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: 627a5365-10e7-4b81-9bcc-400a8e857bf9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:58Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:34:58Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:34:58Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c4db1e6f54e4 namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 860097cf-c107-402b-b98b-cd95c9938b8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-22a3d} kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:34:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:58Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c4db1fa87e7a namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 50c0ae27-fe05-4ace-aa7c-6431c304037d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-22a3d} kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:34:58Z" message: 'Created container: sequence-graph-22a3d' metadata: creationTimestamp: "2026-04-22T19:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:58Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c4db228d8cd7 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 98fdc3c7-5229-47fa-927a-741c12db7875 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-22a3d} kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:34:58Z" message: Started container sequence-graph-22a3d metadata: creationTimestamp: "2026-04-22T19:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:58Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c4db22e47ed5 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: c23be0bb-c7f9-4a78-9ba8-1f577aaa83a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-22a3d} kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Stopping container sequence-graph-22a3d metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c54e150e31cb namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: 8eb3a3ed-597b-4fe9-bb91-1535d308cf44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:43:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-22a3d} kind: Pod name: sequence-graph-22a3d-5498b7b956-qwg8j namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 37a05835-34d1-47b6-a049-99a8222ecc59 kind: Event lastTimestamp: "2026-04-22T19:43:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:39Z" name: sequence-graph-22a3d-5498b7b956-qwg8j.18a8c54ea49b096f namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: efed4777-cc10-4b2a-9277-e9f311de255e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-22a3d-5498b7b956 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 44f7612b-98b6-40b7-9e7e-44dc4f235271 kind: Event lastTimestamp: "2026-04-22T19:34:57Z" message: 'Created pod: sequence-graph-22a3d-5498b7b956-qwg8j' metadata: creationTimestamp: "2026-04-22T19:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:57Z" name: sequence-graph-22a3d-5498b7b956.18a8c4dae1328c24 namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 0cf75329-97bb-4bbe-850c-eab964bd7394 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-22a3d namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 3246b2ca-4192-43b0-acdb-3db13de1a491 kind: Event lastTimestamp: "2026-04-22T19:34:57Z" message: Scaled up replica set sequence-graph-22a3d-5498b7b956 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:57Z" name: sequence-graph-22a3d.18a8c4dae085390d namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 0260714a-145a-4f67-a30b-a7f00dc419e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-22a3d namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 7cdcad44-7b7f-49fc-a6b6-92a4d33b4f03 kind: Event lastTimestamp: "2026-04-22T19:34:57Z" message: InferenceGraph [sequence-graph-22a3d] is Ready metadata: creationTimestamp: "2026-04-22T19:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:34:57Z" name: sequence-graph-22a3d.18a8c4dae54f16f2 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: c1c993bd-dc32-40ce-880d-28eadaff73cf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:35:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-22a3d namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 42012797-01fe-464a-81d0-0255d2110bd1 kind: Event lastTimestamp: "2026-04-22T19:35:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:42Z" name: sequence-graph-22a3d.18a8c4de5ebe967b namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: cfc05c64-aea9-4b5d-bab1-15b063f67bcf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:35:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-22a3d namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 42012797-01fe-464a-81d0-0255d2110bd1 kind: Event lastTimestamp: "2026-04-22T19:35:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:42Z" name: sequence-graph-22a3d.18a8c4de5f1c7db0 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 877d835f-9a06-4f05-9a69-0fb6fccbcd72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:22.361055Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3becf-689b8c8984-dxw8k to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c55e6e04b896 namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 240eae89-ef54-4f2c-ad85-9adc4945251f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c55e86b47cbb namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: ac48a744-47ad-4f00-a094-6790db0a6c71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3becf} kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c55e87f2d96a namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: 8df7a8e2-afe6-471f-8522-f615a8439aea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3becf} kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: 'Created container: sequence-graph-3becf' metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c55e8b63507d namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: 3f9c1722-acb0-4e28-8301-85546710ff01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3becf} kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: Started container sequence-graph-3becf metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c55e8bd310db namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 4a68792c-a4f1-480d-9081-e9565fe8a614 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3becf} kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Stopping container sequence-graph-3becf metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c560c3e15b34 namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: 878b88f5-7fb5-4973-b7a9-f97cd50bc791 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3becf} kind: Pod name: sequence-graph-3becf-689b8c8984-dxw8k namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 4d649f08-d7aa-48fb-b43f-e4275ccbb4cd kind: Event lastTimestamp: "2026-04-22T19:44:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:59Z" name: sequence-graph-3becf-689b8c8984-dxw8k.18a8c561328f20fc namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 507c4bde-65fa-4b58-adaa-e4ceba80dac1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3becf-689b8c8984 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: e4ac1524-0dfe-4699-b3e9-e286431bdf0b kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: 'Created pod: sequence-graph-3becf-689b8c8984-dxw8k' metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf-689b8c8984.18a8c55e6d495b66 namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: f5182e73-2059-4ef0-8499-76483506ac29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3becf namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 686726ca-a3b7-433d-9dda-7c20bb88f776 kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: Scaled up replica set sequence-graph-3becf-689b8c8984 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf.18a8c55e69e96d78 namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: 43286074-18ff-4494-abe5-8be75c55672e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3becf namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: 80a623dd-a9df-4ffb-b00e-b3a8e3787caf kind: Event lastTimestamp: "2026-04-22T19:44:22Z" message: InferenceGraph [sequence-graph-3becf] is Ready metadata: creationTimestamp: "2026-04-22T19:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:22Z" name: sequence-graph-3becf.18a8c55e70da732f namespace: kserve-ci-e2e-test resourceVersion: "20146" uid: 7d0173f0-94a3-4d19-bbc7-cacbcf4d97e7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:45:42.604417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7d5c8-5fbffdc9c5-g5xfd to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:45:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:45:42Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c5711ce54398 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: e82b51aa-2575-4414-9e88-ff6675f91f4c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:43Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: "2026-04-22T19:45:43Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:45:43Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c57135a38e0b namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 03e74fc7-2414-43e3-babe-840cfb4686ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7d5c8} kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: "2026-04-22T19:45:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:43Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c57136d80416 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: 61edf678-2eb2-4a02-b053-8c9cb3505fb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7d5c8} kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: "2026-04-22T19:45:43Z" message: 'Created container: sequence-graph-7d5c8' metadata: creationTimestamp: "2026-04-22T19:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:43Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c5713ac76f79 namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 972b1833-4e85-44f6-b83a-8e1090ff41ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7d5c8} kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: "2026-04-22T19:45:43Z" message: Started container sequence-graph-7d5c8 metadata: creationTimestamp: "2026-04-22T19:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:43Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c5713b78ad86 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: d13173e9-1db3-4381-a9c6-a10c41654658 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7d5c8} kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Stopping container sequence-graph-7d5c8 metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c5e44b0b2028 namespace: kserve-ci-e2e-test resourceVersion: "24571" uid: ee4b74e0-24a2-41f5-9009-4fd684fd4aab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7d5c8} kind: Pod name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: bbc2835a-9e3e-4899-b3cc-1667ea5c7b17 kind: Event lastTimestamp: "2026-04-22T19:54:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:24Z" name: sequence-graph-7d5c8-5fbffdc9c5-g5xfd.18a8c5e4cdfc6c28 namespace: kserve-ci-e2e-test resourceVersion: "24952" uid: 3e0a92c6-9757-4a60-9c64-8c18f4e2c4ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7d5c8-5fbffdc9c5 namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: d189935e-891d-4d24-8372-9e77732ce0b9 kind: Event lastTimestamp: "2026-04-22T19:45:42Z" message: 'Created pod: sequence-graph-7d5c8-5fbffdc9c5-g5xfd' metadata: creationTimestamp: "2026-04-22T19:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:42Z" name: sequence-graph-7d5c8-5fbffdc9c5.18a8c5711c6d76c2 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: a3c297c6-70dd-466d-b6f3-db44e0798dac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: ed6360cd-8fc2-45ff-8fcf-ca985316cf3a kind: Event lastTimestamp: "2026-04-22T19:45:42Z" message: Scaled up replica set sequence-graph-7d5c8-5fbffdc9c5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:42Z" name: sequence-graph-7d5c8.18a8c5711bb73656 namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 3e0c0b63-8543-4b38-98d7-18245b022c07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: e4265cd9-e44d-47d7-abe8-63bd7105ffc3 kind: Event lastTimestamp: "2026-04-22T19:45:42Z" message: InferenceGraph [sequence-graph-7d5c8] is Ready metadata: creationTimestamp: "2026-04-22T19:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:45:42Z" name: sequence-graph-7d5c8.18a8c5711ec27ade namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 919cc326-6a27-444e-9747-ac10a2f22b4e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: abc7acb7-ed5e-44f1-9500-e7dbe4ad70ba kind: Event lastTimestamp: "2026-04-22T19:45:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:57Z" name: sequence-graph-7d5c8.18a8c5749a1fa8ac namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 3268d2a9-8366-4650-b869-bba87a0736b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: abc7acb7-ed5e-44f1-9500-e7dbe4ad70ba kind: Event lastTimestamp: "2026-04-22T19:45:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:57Z" name: sequence-graph-7d5c8.18a8c5749a825606 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 421783ea-9353-43b9-980e-344756cdd1ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: abc7acb7-ed5e-44f1-9500-e7dbe4ad70ba kind: Event lastTimestamp: "2026-04-22T19:46:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:12Z" name: sequence-graph-7d5c8.18a8c578190e0234 namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 202a0a76-4685-421f-98a4-7f66bc8ee859 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: abc7acb7-ed5e-44f1-9500-e7dbe4ad70ba kind: Event lastTimestamp: "2026-04-22T19:46:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:12Z" name: sequence-graph-7d5c8.18a8c5781971740a namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: ef2913eb-b97a-445a-b614-ab675e7dc8ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:55:41.860230Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-0c611-7c9746c8d4-fmcpt to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:55:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:55:41Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c5fca35356d4 namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: 41c80483-c05d-4237-8c8b-031ab26aeefb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T19:55:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-0c611-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:42Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c5fcaf0fab96 namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 78fa375b-5a1f-4aa1-b339-514a8fbcdc6f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25795" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T19:55:42Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:55:42Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c5fcdff251be namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: 91d88391-d751-4b85-985b-6f1d5778a67c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c611} kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T19:55:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:42Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c5fce13e0c1a namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: edff960c-2ae4-456e-b3eb-545e0b783b50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c611} kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T19:55:42Z" message: 'Created container: splitter-graph-0c611' metadata: creationTimestamp: "2026-04-22T19:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:42Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c5fce429f300 namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: d1989134-2b83-41e6-b910-811d13d464cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c611} kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T19:55:42Z" message: Started container splitter-graph-0c611 metadata: creationTimestamp: "2026-04-22T19:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:42Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c5fce49b540b namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: c80c4470-a91e-4142-bc51-c041671ff7d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:03:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c611} kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T20:03:56Z" message: Stopping container splitter-graph-0c611 metadata: creationTimestamp: "2026-04-22T20:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:03:56Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c66fd6e949d4 namespace: kserve-ci-e2e-test resourceVersion: "29519" uid: 1513b9c4-e184-42a0-81ae-703c5e080078 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T20:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c611} kind: Pod name: splitter-graph-0c611-7c9746c8d4-fmcpt namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 79f970de-fcb1-488a-9b4a-84f61a83a17b kind: Event lastTimestamp: "2026-04-22T20:04:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T20:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:04:24Z" name: splitter-graph-0c611-7c9746c8d4-fmcpt.18a8c67076be0819 namespace: kserve-ci-e2e-test resourceVersion: "29749" uid: d5dd53ee-aa8a-486a-bb4e-d6d47a434cf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-0c611-7c9746c8d4 namespace: kserve-ci-e2e-test resourceVersion: "25783" uid: 3d6404db-2367-42e1-9c9d-4baf87945557 kind: Event lastTimestamp: "2026-04-22T19:55:41Z" message: 'Created pod: splitter-graph-0c611-7c9746c8d4-fmcpt' metadata: creationTimestamp: "2026-04-22T19:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:41Z" name: splitter-graph-0c611-7c9746c8d4.18a8c5fca2ef4401 namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: b5212af6-9079-4dfc-9ef2-1bdb8abc81c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 6e1fad12-a7c2-4b1b-adb1-aa9e9a268c03 kind: Event lastTimestamp: "2026-04-22T19:55:41Z" message: Scaled up replica set splitter-graph-0c611-7c9746c8d4 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:41Z" name: splitter-graph-0c611.18a8c5fca211099d namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 33241836-6671-4e55-b673-b2be7abcfb71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 0784d876-8510-4fdf-bfe7-c014439687ed kind: Event lastTimestamp: "2026-04-22T19:55:41Z" message: InferenceGraph [splitter-graph-0c611] is Ready metadata: creationTimestamp: "2026-04-22T19:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:55:41Z" name: splitter-graph-0c611.18a8c5fca6b75eca namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 40218e89-aaf6-473f-981b-c653566d664c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: ee5bb112-ca47-4f50-a1f6-2eb444314867 kind: Event lastTimestamp: "2026-04-22T19:55:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:56Z" name: splitter-graph-0c611.18a8c60020c08d3e namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: 6004dbbc-b0a6-4798-b78f-ea66a21b10f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: ee5bb112-ca47-4f50-a1f6-2eb444314867 kind: Event lastTimestamp: "2026-04-22T19:55:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:56Z" name: splitter-graph-0c611.18a8c600211dfb70 namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 70feae72-5acc-4dbf-b0f1-936057595fdb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:56:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: ee5bb112-ca47-4f50-a1f6-2eb444314867 kind: Event lastTimestamp: "2026-04-22T19:56:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:56:11Z" name: splitter-graph-0c611.18a8c6039fb2a949 namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: 7849d921-144c-4ce9-aeb3-fa1a3f6a3a0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:56:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: ee5bb112-ca47-4f50-a1f6-2eb444314867 kind: Event lastTimestamp: "2026-04-22T19:56:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:56:11Z" name: splitter-graph-0c611.18a8c603a00f564e namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 3d6e2bc7-5126-44e7-a47e-0a0fe08a26d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:54:21.568413Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-44fda-85c85c6697-bn7lk to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5e9f18f6f1c namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: aa77274f-bc0c-4498-b31e-f5484964396f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: "2026-04-22T19:54:21Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5ea0a8aade6 namespace: kserve-ci-e2e-test resourceVersion: "24929" uid: 61687eda-d98b-49d5-ac3c-207201f00668 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44fda} kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: "2026-04-22T19:54:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:22Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5ea0bcd4662 namespace: kserve-ci-e2e-test resourceVersion: "24931" uid: 58867ebe-6a86-459c-b35a-6379b53852c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44fda} kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: "2026-04-22T19:54:22Z" message: 'Created container: splitter-graph-44fda' metadata: creationTimestamp: "2026-04-22T19:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:22Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5ea0f533568 namespace: kserve-ci-e2e-test resourceVersion: "24932" uid: 7ce611c4-537e-4c69-af23-8b78aff361d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44fda} kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: "2026-04-22T19:54:22Z" message: Started container splitter-graph-44fda metadata: creationTimestamp: "2026-04-22T19:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:22Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5ea0fc22d1e namespace: kserve-ci-e2e-test resourceVersion: "24933" uid: d823a0b4-ce1c-4d3f-94a6-f31414616b34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44fda} kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Stopping container splitter-graph-44fda metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:31Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5ec4a958dc6 namespace: kserve-ci-e2e-test resourceVersion: "25045" uid: 67f65e4f-28df-4b2f-94e7-dfa20e25fa6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44fda} kind: Pod name: splitter-graph-44fda-85c85c6697-bn7lk namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 4de03c54-1d6d-4e7c-aed1-8e60f60f4e18 kind: Event lastTimestamp: "2026-04-22T19:54:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:59Z" name: splitter-graph-44fda-85c85c6697-bn7lk.18a8c5ecdae59a19 namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: 3ab55444-ce5d-4404-bd89-71d9e16c9cc2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-44fda-85c85c6697 namespace: kserve-ci-e2e-test resourceVersion: "24904" uid: 06aff50d-11f8-49cd-9957-22d468f75a64 kind: Event lastTimestamp: "2026-04-22T19:54:21Z" message: 'Created pod: splitter-graph-44fda-85c85c6697-bn7lk' metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda-85c85c6697.18a8c5e9f12ec14f namespace: kserve-ci-e2e-test resourceVersion: "24908" uid: 791b3386-caa4-40bd-9af9-bd3a34aa521c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-44fda namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: 46c287c7-8f71-43b4-aef8-0a8f63de6666 kind: Event lastTimestamp: "2026-04-22T19:54:21Z" message: Scaled up replica set splitter-graph-44fda-85c85c6697 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda.18a8c5e9f097f742 namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: a586673f-e59a-48eb-8487-1580d0a02bab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-44fda namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 4f35fa37-ba59-4e3a-ba3a-eeb3c40d12ad kind: Event lastTimestamp: "2026-04-22T19:54:21Z" message: InferenceGraph [splitter-graph-44fda] is Ready metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda.18a8c5e9f4f16e9b namespace: kserve-ci-e2e-test resourceVersion: "24921" uid: 95fd8d31-9963-404a-b1a9-e3fd7ad7b424 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-44fda namespace: kserve-ci-e2e-test resourceVersion: "24894" uid: 4f35fa37-ba59-4e3a-ba3a-eeb3c40d12ad kind: Event lastTimestamp: "2026-04-22T19:54:21Z" message: 'Failed to update status for InferenceGraph "splitter-graph-44fda": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-44fda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda.18a8c5e9f6a31327 namespace: kserve-ci-e2e-test resourceVersion: "24924" uid: 85655c13-f45d-4972-ad1c-2e88f842711f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-44fda namespace: kserve-ci-e2e-test resourceVersion: "24894" uid: 4f35fa37-ba59-4e3a-ba3a-eeb3c40d12ad kind: Event lastTimestamp: "2026-04-22T19:54:21Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-44fda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:21Z" name: splitter-graph-44fda.18a8c5e9f6a360f8 namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: f01eb90b-6092-4e89-a28b-3d9831961503 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:42:26.219848Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0213e-predictor-6b5fb55459-7xg9m to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c5436376981f namespace: kserve-ci-e2e-test resourceVersion: "18819" uid: 14a52daa-788d-4d01-b69e-07ddc8f91d2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c5436a81e51d namespace: kserve-ci-e2e-test resourceVersion: "18847" uid: c8f5dd42-c932-4eaf-a29b-94603a8267e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c5436bd1e719 namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: a90c5a11-ab3e-42d9-91ed-01b6770d65ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c54373e3c787 namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 55343d9a-0d60-455b-9e96-c23e8dd8d2e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c54374571a5e namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: 435380a5-dbde-46da-b6de-e3dfaee281c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: "2026-04-22T19:43:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:47Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c5438a7f7455 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 59ab3be4-638f-4c47-9b49-704ca44ab795 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 5cf7052b-2101-44bd-9c75-b86e4b7314c1 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m.18a8c556133aa3a9 namespace: kserve-ci-e2e-test resourceVersion: "19726" uid: e949f49d-4878-45b4-97bb-3feb6f6b9471 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0213e-predictor-6b5fb55459 namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: ebc790e4-5b90-4292-abc3-acbabd99f6e6 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: 'Created pod: success-200-isvc-0213e-predictor-6b5fb55459-7xg9m' metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor-6b5fb55459.18a8c54363082fd7 namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 07512a17-e239-4bfc-bdb9-acfa3f68be07 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18784" uid: 5cf57e71-5e1b-4079-b94a-25622abfaec4 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Scaled up replica set success-200-isvc-0213e-predictor-6b5fb55459 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-0213e-predictor.18a8c5435f7e6d70 namespace: kserve-ci-e2e-test resourceVersion: "18794" uid: ba5d8627-b148-41b5-b5ea-161bcc93d754 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: 963a801a-0294-409f-8927-707c09cadba1 kind: Event lastTimestamp: "2026-04-22T19:42:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:41Z" name: success-200-isvc-0213e-predictor.18a8c546ddf0c688 namespace: kserve-ci-e2e-test resourceVersion: "18980" uid: 50352521-8125-4061-88ef-1c302e4d77ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: 963a801a-0294-409f-8927-707c09cadba1 kind: Event lastTimestamp: "2026-04-22T19:42:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:42:41Z" name: success-200-isvc-0213e-predictor.18a8c546de373e5b namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: e4d2bede-beef-4933-bdbe-028083205d89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:42:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18981" uid: 963a801a-0294-409f-8927-707c09cadba1 kind: Event lastTimestamp: "2026-04-22T19:43:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:41Z" name: success-200-isvc-0213e-predictor.18a8c54a5d3c5b4d namespace: kserve-ci-e2e-test resourceVersion: "19631" uid: 8b290239-71d8-4ef3-b2f8-609a3958269c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:42:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0213e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18981" uid: 963a801a-0294-409f-8927-707c09cadba1 kind: Event lastTimestamp: "2026-04-22T19:43:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:41Z" name: success-200-isvc-0213e-predictor.18a8c54a5d9d9e4b namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: 91437264-4a49-49fa-934f-32df0229b013 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0213e namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: 1d0870df-6f47-414e-a68a-4b9b37b70970 kind: Event lastTimestamp: "2026-04-22T19:43:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:17Z" name: success-200-isvc-0213e.18a8c5435f536aa7 namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 7589a11b-0899-4d06-916c-07845395fc95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0213e namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: 1d0870df-6f47-414e-a68a-4b9b37b70970 kind: Event lastTimestamp: "2026-04-22T19:43:17Z" message: InferenceService [success-200-isvc-0213e] is Ready metadata: creationTimestamp: "2026-04-22T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:17Z" name: success-200-isvc-0213e.18a8c54f6e206707 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 6932c2cb-0f19-462b-8774-5e8564e2bf3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:54:31.776237Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0c611-predictor-55867b6f47-7ssmc to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c5ec51fe70ce namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: cac7d7c3-fd3b-462a-a4ee-1732eaf417e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c5ec59b69779 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: f55b420d-3800-4c60-820f-0ece4ede369e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c5ec5aeb7a04 namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: c0827b37-4ab9-4c17-9af1-4e2164ee64d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c5ec5f1baca1 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: a00bf6b3-1348-4133-bec3-c02e8757fdbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: "2026-04-22T19:54:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:32Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c5ec5f92458a namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: 485e06b0-741b-43fc-a3d0-1bed3decdab5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: "2026-04-22T19:55:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:13Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c5ec651feed0 namespace: kserve-ci-e2e-test resourceVersion: "25538" uid: 5e97702c-55ad-4fe5-ae1e-88556a82c2ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:03:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 3b998a1e-04b5-400d-9382-8f9ce90e7c1b kind: Event lastTimestamp: "2026-04-22T20:03:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:03:56Z" name: success-200-isvc-0c611-predictor-55867b6f47-7ssmc.18a8c66fdb88f35d namespace: kserve-ci-e2e-test resourceVersion: "29532" uid: b0b07216-c4ab-4e16-8352-57d0edf3769d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0c611-predictor-55867b6f47 namespace: kserve-ci-e2e-test resourceVersion: "25069" uid: a8f43ac0-a33a-4a3c-953b-24c9dfd270c6 kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: 'Created pod: success-200-isvc-0c611-predictor-55867b6f47-7ssmc' metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-0c611-predictor-55867b6f47.18a8c5ec51892ae8 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: a97f3ccc-bd47-4476-943e-24be12b24e2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25065" uid: 6df8e11c-83bb-487f-9050-0516cde2c6dd kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Scaled up replica set success-200-isvc-0c611-predictor-55867b6f47 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-0c611-predictor.18a8c5ec4fe57fb0 namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: 92bfa026-aea6-4b25-887d-8237d3bec2c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: 8c7978e9-8fa5-4e18-b7ed-644136713b06 kind: Event lastTimestamp: "2026-04-22T19:55:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:16Z" name: success-200-isvc-0c611-predictor.18a8c5efcf4fff78 namespace: kserve-ci-e2e-test resourceVersion: "25568" uid: b548babb-7305-4ceb-aa5a-e3cc73afe7b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: 8c7978e9-8fa5-4e18-b7ed-644136713b06 kind: Event lastTimestamp: "2026-04-22T19:55:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:16Z" name: success-200-isvc-0c611-predictor.18a8c5efcfb3e9b4 namespace: kserve-ci-e2e-test resourceVersion: "25569" uid: 1f6c1cfb-6239-44a2-9a12-975ad28ac2c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:55:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 8c7978e9-8fa5-4e18-b7ed-644136713b06 kind: Event lastTimestamp: "2026-04-22T19:56:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:56:01Z" name: success-200-isvc-0c611-predictor.18a8c5fa4bb86ec7 namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 472fc8e0-7861-480e-ab06-23efa750dba9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:55:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c611-predictor namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 8c7978e9-8fa5-4e18-b7ed-644136713b06 kind: Event lastTimestamp: "2026-04-22T19:56:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:56:01Z" name: success-200-isvc-0c611-predictor.18a8c5fa4c1c5fc5 namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: 7bdec862-b2e0-4ed3-af48-1aa87ee24a16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: bcf52963-0ddf-4204-ac6e-125be6a42d14 kind: Event lastTimestamp: "2026-04-22T19:55:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:55:23Z" name: success-200-isvc-0c611.18a8c5ec4fb8cbcc namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: 467c5339-d5ec-4d1c-88f5-75790ca8a500 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0c611 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: bcf52963-0ddf-4204-ac6e-125be6a42d14 kind: Event lastTimestamp: "2026-04-22T19:55:23Z" message: InferenceService [success-200-isvc-0c611] is Ready metadata: creationTimestamp: "2026-04-22T19:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:55:23Z" name: success-200-isvc-0c611.18a8c5f84871648c namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: dccda7bc-a841-4dec-907f-6d25489eebe1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:33:01.327709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c4bfdd4493ba namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: ff1a91a3-6cba-4a51-8c64-24839e0c1001 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c4bfe481e8d8 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 4ff39e11-9651-47eb-875a-716f430266ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c4bfe5f3e0d8 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: a57eb79c-ff99-48ef-9351-75531b26c6cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c4bfed4171ae namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 05da6077-4279-4fcf-88c8-8b42f3e80152 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c4bfedb3e023 namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: a1d390dd-4533-45cf-b8e6-db0672249a8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: "2026-04-22T19:33:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:43Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c4c0101de933 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 7e1d4178-5581-4d74-851d-f1e47fe69b02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 7a370686-f446-4ece-b109-3f444ec79f60 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6.18a8c5435e695cbd namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: a61b17df-4abc-4c8b-9604-08425fd40801 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1be4a-predictor-789bcb6475 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: d2a5a253-3679-4cf6-8384-b2c43f6abcdb kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'Created pod: success-200-isvc-1be4a-predictor-789bcb6475-x8cl6' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor-789bcb6475.18a8c4bfdcbf9647 namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 863b6572-62a3-4e6c-b1de-0fba6823e15d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14349" uid: 17924aca-bbd4-4339-bbde-0354f8981e53 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Scaled up replica set success-200-isvc-1be4a-predictor-789bcb6475 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a-predictor.18a8c4bfd9c5031d namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 78a89b4a-ff07-4882-9c07-1b97a2d2779c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 2cded790-3801-4cc0-97e0-ce62d22e45ea kind: Event lastTimestamp: "2026-04-22T19:33:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:16Z" name: success-200-isvc-1be4a-predictor.18a8c4c3582a9474 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 6a042e27-5b87-4483-b2f5-c140e4bc7f53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 2cded790-3801-4cc0-97e0-ce62d22e45ea kind: Event lastTimestamp: "2026-04-22T19:33:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:16Z" name: success-200-isvc-1be4a-predictor.18a8c4c358913f59 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: e0935764-1d58-404b-8be4-3d8dfd486f0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 2cded790-3801-4cc0-97e0-ce62d22e45ea kind: Event lastTimestamp: "2026-04-22T19:34:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:16Z" name: success-200-isvc-1be4a-predictor.18a8c4c6d729361f namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 2f43f570-b7f9-4b26-84e2-a5ace303ee38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1be4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 2cded790-3801-4cc0-97e0-ce62d22e45ea kind: Event lastTimestamp: "2026-04-22T19:34:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:16Z" name: success-200-isvc-1be4a-predictor.18a8c4c6d78ac343 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 2d07b752-4e64-4920-ae23-4fd7a9e008f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 2cdc3dd4-fd4c-4936-a56a-28397512ae38 kind: Event lastTimestamp: "2026-04-22T19:33:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:53Z" name: success-200-isvc-1be4a.18a8c4bfd99f2ddc namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 485fd4c6-b3d8-45cc-8f6f-affb05f2fa19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 2cdc3dd4-fd4c-4936-a56a-28397512ae38 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-1be4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1be4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a.18a8c4bfda18d63a namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 8fa845d0-a3f6-4681-ab73-8a4ac937cc1a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 2cdc3dd4-fd4c-4936-a56a-28397512ae38 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1be4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-1be4a.18a8c4bfda19385d namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 8f211bdf-ecf0-4fcf-8ce6-9d4f7e0a0b7a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1be4a namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 2cdc3dd4-fd4c-4936-a56a-28397512ae38 kind: Event lastTimestamp: "2026-04-22T19:33:53Z" message: InferenceService [success-200-isvc-1be4a] is Ready metadata: creationTimestamp: "2026-04-22T19:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:53Z" name: success-200-isvc-1be4a.18a8c4cbf3e2ae60 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 9d2cde6a-5a3c-4e78-bcd7-b61350115cd4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:33:47.408644Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-22a3d-predictor-86595f965c-2vstd to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c4ca97e7d7be namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 1508ef65-dac6-4578-8e36-0e437c472a24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c4ca9fd555e3 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: c9afd1fa-2e83-4aae-9383-a36a8f1e8d39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c4caa1886ed8 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 5c845c45-32bd-44d7-85de-3a423d283bda reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c4caa8c8b298 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 0526456d-34e9-4b62-8808-e66274e8c4e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c4caa9275f87 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: d01d29ae-6444-4bcf-89ba-c3585234c43c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: "2026-04-22T19:34:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:29Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c4cacebcf0e6 namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: a30c140c-63f8-4ae4-9de3-e93aa9c09541 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22a3d-predictor-86595f965c-2vstd namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 4f2cc0be-1178-469d-ba40-8ca28096298a kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-22a3d-predictor-86595f965c-2vstd.18a8c54e1a60031a namespace: kserve-ci-e2e-test resourceVersion: "19265" uid: 8addd8eb-5bcc-4515-932e-b2cbf28afde7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-22a3d-predictor-86595f965c namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 0b636ccf-b92e-43bd-9333-1d8861fa70d9 kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: 'Created pod: success-200-isvc-22a3d-predictor-86595f965c-2vstd' metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor-86595f965c.18a8c4ca977ac725 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 17081cd7-d91c-43b0-a1fb-284688457313 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-22a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: f94cc1ce-eea9-41fd-98c9-bf264564e5ba kind: Event lastTimestamp: "2026-04-22T19:33:47Z" message: Scaled up replica set success-200-isvc-22a3d-predictor-86595f965c from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:47Z" name: success-200-isvc-22a3d-predictor.18a8c4ca953316ea namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: bf97c055-89b6-4ac1-8215-ec8ea1226621 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:34:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-22a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 0cf44f79-0e63-4cb8-9c29-77a5076be8f9 kind: Event lastTimestamp: "2026-04-22T19:35:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:02Z" name: success-200-isvc-22a3d-predictor.18a8c4ce13a1090c namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: c7bdd4b6-fd8f-4e7d-9273-5f4a922b738c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:34:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-22a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 0cf44f79-0e63-4cb8-9c29-77a5076be8f9 kind: Event lastTimestamp: "2026-04-22T19:35:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:02Z" name: success-200-isvc-22a3d-predictor.18a8c4ce140f8414 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: d0bb9398-175d-460d-a1e2-8ad4eeda3a97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-22a3d namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: ef604a98-0832-4124-bbc2-f5ec8d39ade8 kind: Event lastTimestamp: "2026-04-22T19:34:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:34:39Z" name: success-200-isvc-22a3d.18a8c4ca94f548a6 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 37ba8b8e-d854-46d7-815f-a9f56b11fb20 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-22a3d namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: ef604a98-0832-4124-bbc2-f5ec8d39ade8 kind: Event lastTimestamp: "2026-04-22T19:34:39Z" message: InferenceService [success-200-isvc-22a3d] is Ready metadata: creationTimestamp: "2026-04-22T19:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:34:39Z" name: success-200-isvc-22a3d.18a8c4d6b20bf928 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: e6a57e38-8157-4abb-a6a4-bb0e668f9074 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:43:12.266652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19269" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3becf-predictor-5fd4d7b59-qf97h to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c54e1c11099a namespace: kserve-ci-e2e-test resourceVersion: "19275" uid: 99604685-b546-438c-9a05-9a44c09e2ed2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c54e233ab881 namespace: kserve-ci-e2e-test resourceVersion: "19322" uid: 629ae0df-38de-4886-b9d6-07e1a0fe93f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c54e248b40d4 namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: 5a0f2372-e77c-4d3c-861c-120c0a7db603 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c54e299bef44 namespace: kserve-ci-e2e-test resourceVersion: "19329" uid: e814f033-dfe4-4232-a03f-7b512c18945f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c54e2a0b21cb namespace: kserve-ci-e2e-test resourceVersion: "19331" uid: 5f00ff6c-30ba-4d22-8dd4-cc7334ef70b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:43:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: "2026-04-22T19:44:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:43:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:34Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c54e48ce6236 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: de8880aa-190a-42e6-a916-84940530a91f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: 678037b8-563b-4586-b25c-1d27ecdbfcaa kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h.18a8c560c9ced2ea namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 03521e07-2a7d-4283-93d2-44fe38d3869e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3becf-predictor-5fd4d7b59 namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: adae5833-b94f-4027-a575-c23b80cfea7a kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'Created pod: success-200-isvc-3becf-predictor-5fd4d7b59-qf97h' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor-5fd4d7b59.18a8c54e1b7b6511 namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: f62e1973-4880-461a-8eed-be036b40ce6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19255" uid: 650b6f92-9598-44f7-ae18-cacf533c7fce kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: Scaled up replica set success-200-isvc-3becf-predictor-5fd4d7b59 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf-predictor.18a8c54e19fa9ec2 namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: bc05889a-7420-4a7b-b902-a2e90358b4d5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19260" uid: a72a882b-9f6f-47a7-a360-aa943bae1e8c kind: Event lastTimestamp: "2026-04-22T19:43:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:42Z" name: success-200-isvc-3becf-predictor.18a8c55198b8cc2f namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: a67201a6-0a58-4f95-8466-420dcd241882 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19260" uid: a72a882b-9f6f-47a7-a360-aa943bae1e8c kind: Event lastTimestamp: "2026-04-22T19:43:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:42Z" name: success-200-isvc-3becf-predictor.18a8c551991b68b2 namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: 2aa2ca89-8782-4dd4-a6f6-1ac4d1c66f3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a72a882b-9f6f-47a7-a360-aa943bae1e8c kind: Event lastTimestamp: "2026-04-22T19:44:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:27Z" name: success-200-isvc-3becf-predictor.18a8c55896e2586e namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: a347a21c-e37b-4752-bff6-433ad2443fc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3becf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a72a882b-9f6f-47a7-a360-aa943bae1e8c kind: Event lastTimestamp: "2026-04-22T19:44:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:27Z" name: success-200-isvc-3becf-predictor.18a8c558973f318d namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: c47972c6-5d27-49c9-80a4-0e03b75c6bef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 8615ad5a-2603-4a20-9018-a09a5f92b657 kind: Event lastTimestamp: "2026-04-22T19:44:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:04Z" name: success-200-isvc-3becf.18a8c54e1a355cf7 namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: 7c3c840c-d405-4b34-b51f-8eaa7ec4d658 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 8615ad5a-2603-4a20-9018-a09a5f92b657 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-3becf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3becf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf.18a8c54e1a8b0d6d namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: 5d0232ea-0bad-4338-8a39-7705676508c1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 8615ad5a-2603-4a20-9018-a09a5f92b657 kind: Event lastTimestamp: "2026-04-22T19:43:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3becf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:12Z" name: success-200-isvc-3becf.18a8c54e1a8b6efb namespace: kserve-ci-e2e-test resourceVersion: "19268" uid: 94de5b08-ea95-4bd6-8498-16db94f7daeb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3becf namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: 8615ad5a-2603-4a20-9018-a09a5f92b657 kind: Event lastTimestamp: "2026-04-22T19:44:04Z" message: InferenceService [success-200-isvc-3becf] is Ready metadata: creationTimestamp: "2026-04-22T19:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:04Z" name: success-200-isvc-3becf.18a8c55a2c8c8e95 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: f1917f8f-bcef-4bc7-aec3-47fb130ba619 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:53:11.500837Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24148" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5d9a1351385 namespace: kserve-ci-e2e-test resourceVersion: "24159" uid: 814ec592-bed1-49d2-ac87-9073aed3f4a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24163" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5d9a8453e91 namespace: kserve-ci-e2e-test resourceVersion: "24186" uid: 0bda74da-108e-45cc-9a48-9146ed6089cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5d9a972aa19 namespace: kserve-ci-e2e-test resourceVersion: "24189" uid: 05d827ac-72df-4131-b265-6154dc679a06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5d9acb59ff3 namespace: kserve-ci-e2e-test resourceVersion: "24191" uid: e9196ca7-74fb-4def-86a1-f474f1ab74be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5d9ad354d72 namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: ae283f34-4a80-4ef1-b728-11b420baddf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: "2026-04-22T19:54:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:32Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5d9b5138c41 namespace: kserve-ci-e2e-test resourceVersion: "25158" uid: 7c6653f3-3ce8-4c87-bf94-89d5599cea06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: ab3fc4e2-e666-4f3e-aa46-b53556f14131 kind: Event lastTimestamp: "2026-04-22T19:54:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:31Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8.18a8c5ec52b29210 namespace: kserve-ci-e2e-test resourceVersion: "25096" uid: 1ae7c3b2-fe42-4a2d-9570-68d7aad50144 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-44fda-predictor-7b4b7f6c87 namespace: kserve-ci-e2e-test resourceVersion: "24125" uid: 130d3758-5dad-41f0-b5a4-f700e4750e7b kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: 'Created pod: success-200-isvc-44fda-predictor-7b4b7f6c87-bs9s8' metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor-7b4b7f6c87.18a8c5d9a0c9f96d namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 3f0a498e-d08d-4898-bbda-376ff2d78a28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24122" uid: 5c2ca819-87f4-4e1b-be05-4a3a49182944 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Scaled up replica set success-200-isvc-44fda-predictor-7b4b7f6c87 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-44fda-predictor.18a8c5d99d6b2d1b namespace: kserve-ci-e2e-test resourceVersion: "24129" uid: 24550489-18bf-403d-8e5b-437650a1b02c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 56be413c-9948-4e59-b669-52535708b5af kind: Event lastTimestamp: "2026-04-22T19:53:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:41Z" name: success-200-isvc-44fda-predictor.18a8c5dd1c213a77 namespace: kserve-ci-e2e-test resourceVersion: "24436" uid: cd2eef65-de0f-4512-a215-5045ff56d588 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 56be413c-9948-4e59-b669-52535708b5af kind: Event lastTimestamp: "2026-04-22T19:53:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:41Z" name: success-200-isvc-44fda-predictor.18a8c5dd1c9f26d6 namespace: kserve-ci-e2e-test resourceVersion: "24437" uid: 9699daf0-483b-4bc7-b307-44ac3f35daf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24322" uid: 56be413c-9948-4e59-b669-52535708b5af kind: Event lastTimestamp: "2026-04-22T19:54:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:26Z" name: success-200-isvc-44fda-predictor.18a8c5e41a2e277d namespace: kserve-ci-e2e-test resourceVersion: "24969" uid: 740c4e90-8deb-413c-8276-607b31e9b0d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44fda-predictor namespace: kserve-ci-e2e-test resourceVersion: "24322" uid: 56be413c-9948-4e59-b669-52535708b5af kind: Event lastTimestamp: "2026-04-22T19:54:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:54:26Z" name: success-200-isvc-44fda-predictor.18a8c5e41a913c92 namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: d890b15f-d72b-4318-a60a-157387bfee10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44fda namespace: kserve-ci-e2e-test resourceVersion: "24104" uid: 05c867f5-b890-43a3-a033-6096270fb645 kind: Event lastTimestamp: "2026-04-22T19:54:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:02Z" name: success-200-isvc-44fda.18a8c5d99d4328c3 namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 27089e4c-77d5-40fa-a295-fb22bd4fd72c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44fda namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: 05c867f5-b890-43a3-a033-6096270fb645 kind: Event lastTimestamp: "2026-04-22T19:54:02Z" message: InferenceService [success-200-isvc-44fda] is Ready metadata: creationTimestamp: "2026-04-22T19:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:02Z" name: success-200-isvc-44fda.18a8c5e598a2d008 namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: b4cbc581-b1eb-42d9-9cec-3356c412aed9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:43:46.527465Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-62743-predictor-56cb884845-x67tw to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c556162b984f namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: cfefacb9-c5d4-407f-b1a5-ee819065cfd3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c5561d86ac8f namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: 6e140071-529b-412b-9991-3536e8e5990d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c5561eac80a5 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 362dfddf-a2d1-40a9-a0e0-a8ec351f4e3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c556262b9660 namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: 3d882949-16de-4be1-b421-594c0f84f9cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c55626a019b8 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 42112615-3978-4566-b60c-6192cded4e3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:43:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: "2026-04-22T19:44:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:28Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c55639e42f1c namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 37dba31e-45b8-493d-b1f1-996523a5289b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62743-predictor-56cb884845-x67tw namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 373bc30e-6149-48ca-a2be-a953653225a9 kind: Event lastTimestamp: "2026-04-22T19:53:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:11Z" name: success-200-isvc-62743-predictor-56cb884845-x67tw.18a8c5d99cf09435 namespace: kserve-ci-e2e-test resourceVersion: "24123" uid: c8677796-4fc0-4b93-95d4-81eb5ddf82fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-62743-predictor-56cb884845 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 5ea36c21-a1f0-4a30-950c-b8feb6e54cc6 kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: 'Created pod: success-200-isvc-62743-predictor-56cb884845-x67tw' metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor-56cb884845.18a8c556159e7676 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 16cfac09-685e-450b-b69a-2b6be124ad88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 5a16c0b3-b1c5-4ab3-902d-e500274d6b4b kind: Event lastTimestamp: "2026-04-22T19:43:46Z" message: Scaled up replica set success-200-isvc-62743-predictor-56cb884845 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:46Z" name: success-200-isvc-62743-predictor.18a8c55613c60c43 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: d4ab57a5-7a1b-4f90-96bb-da649f7c6f43 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: 899de0e7-b7bf-48cb-b675-aed5a5bc67f7 kind: Event lastTimestamp: "2026-04-22T19:45:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:16Z" name: success-200-isvc-62743-predictor.18a8c559924b1ac4 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: c011cb36-a640-4bc0-878a-efd754e17854 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: 899de0e7-b7bf-48cb-b675-aed5a5bc67f7 kind: Event lastTimestamp: "2026-04-22T19:45:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:16Z" name: success-200-isvc-62743-predictor.18a8c55992b0dd33 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 7619295c-bbaa-4851-a1e3-7d9637db1e48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 899de0e7-b7bf-48cb-b675-aed5a5bc67f7 kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:46Z" name: success-200-isvc-62743-predictor.18a8c55d115901a6 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 642c7138-3f3f-4181-bed2-4299751fc14c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62743-predictor namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 899de0e7-b7bf-48cb-b675-aed5a5bc67f7 kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:46Z" name: success-200-isvc-62743-predictor.18a8c55d11bd0995 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 98bd1ddc-2cfe-4fd7-9cae-4b8c06540b51 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:43:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-62743 namespace: kserve-ci-e2e-test resourceVersion: "19707" uid: 511a2a8c-a166-4609-9f4d-a9125d09b561 kind: Event lastTimestamp: "2026-04-22T19:44:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:38Z" name: success-200-isvc-62743.18a8c55613bd3d25 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: cc55906d-475e-49b8-a5b9-8aedb047ff4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-62743 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: 511a2a8c-a166-4609-9f4d-a9125d09b561 kind: Event lastTimestamp: "2026-04-22T19:44:38Z" message: InferenceService [success-200-isvc-62743] is Ready metadata: creationTimestamp: "2026-04-22T19:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:38Z" name: success-200-isvc-62743.18a8c5621d58fdb9 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 8be0938a-3983-4a4e-8114-c440d11c2dfb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:53:57.431751Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-65828-predictor-b79ff5787-mv5m4 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c5e452e73782 namespace: kserve-ci-e2e-test resourceVersion: "24615" uid: ba094acf-70cc-4833-b6eb-be60a3558b10 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24621" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c5e45a2484e2 namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: f31c2d76-c3da-4173-8756-3fd7830af1d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c5e45b541648 namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: ca7861c8-9394-405a-b20b-0e4cc3f1b5b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c5e462999535 namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: 6c33dca5-7b11-4741-a5a7-6dd6b60369ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c5e46321b6c7 namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: db655dd3-4d07-4c5c-9ade-12a1e6916550 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:53:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T19:54:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:54:39Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c5e4afb172be namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: 54b0ef50-142a-4d36-8803-59e1191a33de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T20:11:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:26Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c6d8abe94e74 namespace: kserve-ci-e2e-test resourceVersion: "32656" uid: ff406c19-2630-4673-88ad-0a60a47a5b66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65828-predictor-b79ff5787-mv5m4 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: e12c4ff3-d289-4622-a303-df834301b56f kind: Event lastTimestamp: "2026-04-22T20:11:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: i/o timeout' metadata: creationTimestamp: "2026-04-22T20:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:30Z" name: success-200-isvc-65828-predictor-b79ff5787-mv5m4.18a8c6d99ffb8c5a namespace: kserve-ci-e2e-test resourceVersion: "32736" uid: 88512039-2f67-4461-8360-b9b95b959000 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-65828-predictor-b79ff5787 namespace: kserve-ci-e2e-test resourceVersion: "24593" uid: d85158b0-23c5-47da-93af-f0ce245f04d3 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: 'Created pod: success-200-isvc-65828-predictor-b79ff5787-mv5m4' metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor-b79ff5787.18a8c5e45266d194 namespace: kserve-ci-e2e-test resourceVersion: "24612" uid: 58d44798-1e9b-4a91-9042-50537b1c9a3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24590" uid: cf316811-07e3-4ce8-bf24-ac1a07b9ec3c kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Scaled up replica set success-200-isvc-65828-predictor-b79ff5787 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-65828-predictor.18a8c5e45091fcad namespace: kserve-ci-e2e-test resourceVersion: "24594" uid: 24b4ded2-7651-4218-bf92-5e6a4b194060 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:54:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24592" uid: 0cfede04-9cee-4b53-ad34-3c4fd701158f kind: Event lastTimestamp: "2026-04-22T19:55:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:12Z" name: success-200-isvc-65828-predictor.18a8c5e7cf3b64cc namespace: kserve-ci-e2e-test resourceVersion: "25530" uid: 77b851f0-2c4f-4c6d-b0d4-3bd068f5a143 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:54:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24592" uid: 0cfede04-9cee-4b53-ad34-3c4fd701158f kind: Event lastTimestamp: "2026-04-22T19:55:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:12Z" name: success-200-isvc-65828-predictor.18a8c5e7cf9b8584 namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 1b5d5995-07ee-4893-8ccc-3f6b583140b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24821" uid: 0cfede04-9cee-4b53-ad34-3c4fd701158f kind: Event lastTimestamp: "2026-04-22T19:55:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:27Z" name: success-200-isvc-65828-predictor.18a8c5eecceb6797 namespace: kserve-ci-e2e-test resourceVersion: "25668" uid: eaa6d791-48e1-48b2-96ad-d391949adbb6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65828-predictor namespace: kserve-ci-e2e-test resourceVersion: "24821" uid: 0cfede04-9cee-4b53-ad34-3c4fd701158f kind: Event lastTimestamp: "2026-04-22T19:55:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:27Z" name: success-200-isvc-65828-predictor.18a8c5eecd4c451a namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: a4fc6e51-0fce-417f-9689-f96deb1d12b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65828 namespace: kserve-ci-e2e-test resourceVersion: "24573" uid: fd1f49f1-cfa6-424a-bdba-5ffb1941732e kind: Event lastTimestamp: "2026-04-22T19:54:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:50Z" name: success-200-isvc-65828.18a8c5e45097f464 namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: 1cd8cc1a-2edc-4181-935a-ac6f3dca6271 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:54:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65828 namespace: kserve-ci-e2e-test resourceVersion: "25316" uid: fd1f49f1-cfa6-424a-bdba-5ffb1941732e kind: Event lastTimestamp: "2026-04-22T19:54:50Z" message: InferenceService [success-200-isvc-65828] is Ready metadata: creationTimestamp: "2026-04-22T19:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:54:50Z" name: success-200-isvc-65828.18a8c5f0934eabd8 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: eb82e733-706a-4d47-a0db-5bbb89e9829f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:32.507564Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7d5c8-predictor-db8d55f47-79f75 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c560cacc2b06 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: be13d6ab-c646-4f67-981b-07cb8af19985 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c560d2d5eae5 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 737206c3-80e8-4a6d-8860-f73b636e2172 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c560d40726c9 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 08c1b89a-eecd-4133-b034-00fc1280c701 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c560dc00759b namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: b30412d6-4f1a-4e90-8e1e-1cd55ccdcdee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c560dc6f3991 namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: e3749e85-62e7-4974-ba7a-096e7aca462b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: "2026-04-22T19:45:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:14Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c560f852401d namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: 116bd2e5-49c6-4f99-bc30-bb40e50c7bce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f57444f4-bd7f-44c3-b762-9278b7dc2927 kind: Event lastTimestamp: "2026-04-22T19:53:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:53:57Z" name: success-200-isvc-7d5c8-predictor-db8d55f47-79f75.18a8c5e473cc48ee namespace: kserve-ci-e2e-test resourceVersion: "24671" uid: 3056c3c6-e1f8-4251-b955-13eb4a005690 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7d5c8-predictor-db8d55f47 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 34583849-d2e1-4a6f-8514-823a34003725 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'Created pod: success-200-isvc-7d5c8-predictor-db8d55f47-79f75' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor-db8d55f47.18a8c560caac9696 namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: c8555722-f55d-4498-a091-3571c11a89f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 8ae75a16-0559-4fd5-a056-5fc20605de7b kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: Scaled up replica set success-200-isvc-7d5c8-predictor-db8d55f47 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8-predictor.18a8c560c8cdb5e7 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 0f963299-749c-4f00-b841-300d13e4093b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 24866a01-f6f5-4c7b-8574-7561bc2d0ffe kind: Event lastTimestamp: "2026-04-22T19:45:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:02Z" name: success-200-isvc-7d5c8-predictor.18a8c564476f8515 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: f9976407-17ae-433a-adce-8c12425008ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 24866a01-f6f5-4c7b-8574-7561bc2d0ffe kind: Event lastTimestamp: "2026-04-22T19:45:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:02Z" name: success-200-isvc-7d5c8-predictor.18a8c56447ec2a2f namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 2a0c0ddd-1c39-46dc-a186-81f3c3bc8129 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 24866a01-f6f5-4c7b-8574-7561bc2d0ffe kind: Event lastTimestamp: "2026-04-22T19:45:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:32Z" name: success-200-isvc-7d5c8-predictor.18a8c56b45281a0f namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 2d399803-e84d-47bf-8937-ab264c95b60e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d5c8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 24866a01-f6f5-4c7b-8574-7561bc2d0ffe kind: Event lastTimestamp: "2026-04-22T19:45:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:32Z" name: success-200-isvc-7d5c8-predictor.18a8c56b4589e46a namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 3f6557a4-f724-40be-bf68-94da77d8b0de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 64bee572-00ff-433f-a9a6-d65b3987a303 kind: Event lastTimestamp: "2026-04-22T19:45:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:45:24Z" name: success-200-isvc-7d5c8.18a8c560c8d59989 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: c2f7a6df-65e6-43cf-82b7-b0afe59d0afb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 64bee572-00ff-433f-a9a6-d65b3987a303 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-7d5c8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7d5c8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8.18a8c560c979db5d namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: f901ceed-0eff-4242-8e96-785d5e2cdb8f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 64bee572-00ff-433f-a9a6-d65b3987a303 kind: Event lastTimestamp: "2026-04-22T19:44:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7d5c8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:32Z" name: success-200-isvc-7d5c8.18a8c560c97a3310 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: aa3773e4-c55f-4356-83b4-5bfe1c6e22aa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d5c8 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 64bee572-00ff-433f-a9a6-d65b3987a303 kind: Event lastTimestamp: "2026-04-22T19:45:24Z" message: InferenceService [success-200-isvc-7d5c8] is Ready metadata: creationTimestamp: "2026-04-22T19:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:45:24Z" name: success-200-isvc-7d5c8.18a8c56cdc2ed5a0 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: d3b1f421-79eb-42ba-87f2-6d404c04f639 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:31:26.917766Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13268" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:31:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:31:26Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4a9e1ff3132 namespace: kserve-ci-e2e-test resourceVersion: "13275" uid: 0e7b2dd1-5d35-4b90-8608-9bc358d789d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13281" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:31:27Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4a9e8928a9b namespace: kserve-ci-e2e-test resourceVersion: "13287" uid: 09c8d4db-6c57-4edb-a893-d4c423126e9d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:31:27Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" metadata: creationTimestamp: "2026-04-22T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:27Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4a9e9c2815a namespace: kserve-ci-e2e-test resourceVersion: "13290" uid: 6b6fd0e2-f5cf-4763-bdd9-53c4b099cb12 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:31:41Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1256" in 14.045s (14.045s including waiting). Image size: 1334198883 bytes.' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:41Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4ad2eee0f84 namespace: kserve-ci-e2e-test resourceVersion: "13566" uid: 69d50273-6ca9-431b-8c91-5e480d1b5dcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:31:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:41Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4ad35057ad0 namespace: kserve-ci-e2e-test resourceVersion: "13568" uid: b9193085-d696-4661-b62d-c1148dc74413 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:31:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:31:41Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4ad35873a0b namespace: kserve-ci-e2e-test resourceVersion: "13569" uid: bfdafae5-e40e-407a-8628-79e01adefe4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:33:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:02Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4ad610eaa02 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 0a1d34c9-90b8-4e9e-8436-b7986dcb90cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 02a881ab-2edc-475b-bdaf-2dc283b1b9e9 kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt.18a8c4bfda36ca4b namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: b01b8d58-8196-4fff-8008-1df2b7d013fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2ca6-predictor-5c9ff77945 namespace: kserve-ci-e2e-test resourceVersion: "13258" uid: 8eded7ae-ac33-443f-8e33-1ce612ff0781 kind: Event lastTimestamp: "2026-04-22T19:31:26Z" message: 'Created pod: success-200-isvc-b2ca6-predictor-5c9ff77945-6n7qt' metadata: creationTimestamp: "2026-04-22T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:26Z" name: success-200-isvc-b2ca6-predictor-5c9ff77945.18a8c4a9e1b9c6d5 namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: 3c82030b-3203-44f2-883d-2e858581d34a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13256" uid: 8b651f52-5ec5-4d76-8caf-19d306af3a85 kind: Event lastTimestamp: "2026-04-22T19:31:26Z" message: Scaled up replica set success-200-isvc-b2ca6-predictor-5c9ff77945 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:26Z" name: success-200-isvc-b2ca6-predictor.18a8c4a9db5a8aba namespace: kserve-ci-e2e-test resourceVersion: "13259" uid: 82f29117-1ecf-4785-a28c-51043c447923 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: ccb78f85-b714-4f7e-acc1-db1d9275ef9b kind: Event lastTimestamp: "2026-04-22T19:31:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:56Z" name: success-200-isvc-b2ca6-predictor.18a8c4ad6129be58 namespace: kserve-ci-e2e-test resourceVersion: "13712" uid: 47e39f6d-2bac-45dc-a0f5-e11f55a1693e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: ccb78f85-b714-4f7e-acc1-db1d9275ef9b kind: Event lastTimestamp: "2026-04-22T19:31:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:31:56Z" name: success-200-isvc-b2ca6-predictor.18a8c4ad6189c0dc namespace: kserve-ci-e2e-test resourceVersion: "13713" uid: c408e5d8-27c6-4071-aaf6-55ad119bd2e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:32:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: ccb78f85-b714-4f7e-acc1-db1d9275ef9b kind: Event lastTimestamp: "2026-04-22T19:32:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:57Z" name: success-200-isvc-b2ca6-predictor.18a8c4b45f81e7af namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 81d4cd73-72a3-4266-a8e6-b879e687f376 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:32:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ca6-predictor namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: ccb78f85-b714-4f7e-acc1-db1d9275ef9b kind: Event lastTimestamp: "2026-04-22T19:32:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:57Z" name: success-200-isvc-b2ca6-predictor.18a8c4b45fe7bbed namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 5a5f4684-30d4-45d7-a5a2-dc2c3a49c055 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "13248" uid: 65d4babd-66de-460e-8b4a-01786e82af0d kind: Event lastTimestamp: "2026-04-22T19:32:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:33Z" name: success-200-isvc-b2ca6.18a8c4a9e1785e1a namespace: kserve-ci-e2e-test resourceVersion: "14034" uid: ec2ec28a-2334-4e34-9049-37bb8b506bc2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "13248" uid: 65d4babd-66de-460e-8b4a-01786e82af0d kind: Event lastTimestamp: "2026-04-22T19:31:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-b2ca6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2ca6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:31:26Z" name: success-200-isvc-b2ca6.18a8c4a9e1c712d1 namespace: kserve-ci-e2e-test resourceVersion: "13274" uid: e7a7180a-6744-49ec-8327-a334fb22022f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "13248" uid: 65d4babd-66de-460e-8b4a-01786e82af0d kind: Event lastTimestamp: "2026-04-22T19:31:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2ca6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:31:26Z" name: success-200-isvc-b2ca6.18a8c4a9e1c7641d namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: 9ff50365-5696-4542-a4ef-ef00ee93c3ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "14032" uid: 65d4babd-66de-460e-8b4a-01786e82af0d kind: Event lastTimestamp: "2026-04-22T19:32:32Z" message: InferenceService [success-200-isvc-b2ca6] is Ready metadata: creationTimestamp: "2026-04-22T19:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:32Z" name: success-200-isvc-b2ca6.18a8c4b9447129df namespace: kserve-ci-e2e-test resourceVersion: "14033" uid: 7c7f1a22-aafb-441c-b39d-97fdc32a141a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:34:11.397998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1be4a-d6b9df876-tkptz to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c4d02dc84529 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 6b1f106b-4bd6-4758-8de4-e6d9050553a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c4d046733759 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: ec0e1de4-5fb6-4ab0-8122-30a572b31888 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1be4a} kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c4d047accc14 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 26b4f422-50df-4f43-b271-4abd2c3fb1e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1be4a} kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: 'Created container: switch-graph-1be4a' metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c4d04a4c51e4 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 7f338a3b-d477-471b-9e97-66551ca0e041 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1be4a} kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: Started container switch-graph-1be4a metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c4d04aaa38d0 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 137c8377-61b9-4cf1-88ae-e186a7ca3b80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1be4a} kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: "2026-04-22T19:42:26Z" message: Stopping container switch-graph-1be4a metadata: creationTimestamp: "2026-04-22T19:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:26Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c54359468fc4 namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 5d165c9a-e6b5-47c6-b9ff-d8f835ad0291 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1be4a} kind: Pod name: switch-graph-1be4a-d6b9df876-tkptz namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 06a1fe01-7206-49cb-9307-29235a9d17e8 kind: Event lastTimestamp: "2026-04-22T19:42:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:42:53Z" name: switch-graph-1be4a-d6b9df876-tkptz.18a8c543e63d7bce namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: 046772ae-7ca2-4176-9394-f4ac69c73663 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1be4a-d6b9df876 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 7ac80a5b-cc01-4604-b67e-8274310ea099 kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: 'Created pod: switch-graph-1be4a-d6b9df876-tkptz' metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a-d6b9df876.18a8c4d02d52926c namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 3977ae15-51bd-4e78-b845-9fd3630e03d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1be4a namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: 6a65b323-9f19-4571-9966-0e6592da9cc2 kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: Scaled up replica set switch-graph-1be4a-d6b9df876 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a.18a8c4d02c72e7d7 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: 3cb082e4-2352-418c-9f8d-e43c16a64182 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1be4a namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 8a9b6f99-7ebd-4418-aa92-963c47a3992c kind: Event lastTimestamp: "2026-04-22T19:34:11Z" message: InferenceGraph [switch-graph-1be4a] is Ready metadata: creationTimestamp: "2026-04-22T19:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:34:11Z" name: switch-graph-1be4a.18a8c4d0319c5269 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 47d058da-9b89-4b04-8116-6f7aacda22a7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:34:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1be4a namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: ef03a625-4d26-440d-80f8-98c6893ceaab kind: Event lastTimestamp: "2026-04-22T19:34:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:41Z" name: switch-graph-1be4a.18a8c4d3ab459ff8 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: a28867f8-4e82-40e9-81d0-a536e03c683a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:34:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1be4a namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: ef03a625-4d26-440d-80f8-98c6893ceaab kind: Event lastTimestamp: "2026-04-22T19:34:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:41Z" name: switch-graph-1be4a.18a8c4d3aba544cf namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 4202dc22-ea74-4d15-933f-d06f8de48766 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:55:07.523648Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-65828-75747dbbc6-px2gk to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:55:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:55:07Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c5f4a4b4a18f namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: c51d587f-d9da-4b78-b6fc-6afb5ee0e184 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:55:07Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T19:55:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-65828-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:08Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c5f4acae97e4 namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 715c4ff3-e93a-4944-af05-d55180cff875 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T19:55:09Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:55:09Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c5f517170931 namespace: kserve-ci-e2e-test resourceVersion: "25499" uid: 2e811011-d8f5-43de-a68d-f0db59167d16 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65828} kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T19:55:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:09Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c5f518432ecd namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: fd1811b5-ee58-4f0a-a5d6-1bc6ebd17e40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65828} kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T19:55:09Z" message: 'Created container: switch-graph-65828' metadata: creationTimestamp: "2026-04-22T19:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:09Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c5f51ba823b3 namespace: kserve-ci-e2e-test resourceVersion: "25503" uid: bf3c8ac8-08cb-41f4-a298-67e8d9eed812 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65828} kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T19:55:09Z" message: Started container switch-graph-65828 metadata: creationTimestamp: "2026-04-22T19:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:55:09Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c5f51c137459 namespace: kserve-ci-e2e-test resourceVersion: "25504" uid: 03c37cae-d9ec-4ccf-927d-8a96edd8008b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65828} kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T20:11:26Z" message: Stopping container switch-graph-65828 metadata: creationTimestamp: "2026-04-22T20:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:26Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c6d8a5e0c4e3 namespace: kserve-ci-e2e-test resourceVersion: "32642" uid: 4dcc0031-9903-49cd-95db-1aec60f4c557 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65828} kind: Pod name: switch-graph-65828-75747dbbc6-px2gk namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: a6d78a95-b6d5-4c2a-a1cc-4d9b25743619 kind: Event lastTimestamp: "2026-04-22T20:11:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T20:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:46Z" name: switch-graph-65828-75747dbbc6-px2gk.18a8c6d9ae1ee86e namespace: kserve-ci-e2e-test resourceVersion: "32844" uid: d06bf187-5efd-4d8a-b761-80f866b3cb32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-65828-75747dbbc6 namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: 39e403cd-79da-4344-ae88-0fc01ebdbc06 kind: Event lastTimestamp: "2026-04-22T19:55:07Z" message: 'Created pod: switch-graph-65828-75747dbbc6-px2gk' metadata: creationTimestamp: "2026-04-22T19:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:07Z" name: switch-graph-65828-75747dbbc6.18a8c5f4a44e1480 namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: deb20011-be94-48e9-8a2f-dbcb93896fda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-65828 namespace: kserve-ci-e2e-test resourceVersion: "25464" uid: 79f34ad2-f0da-46ef-8531-6c79e30f0543 kind: Event lastTimestamp: "2026-04-22T19:55:07Z" message: Scaled up replica set switch-graph-65828-75747dbbc6 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:07Z" name: switch-graph-65828.18a8c5f4a399ba93 namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: ee355cc7-ac2d-49bb-bdd9-89b816c5ecc7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-65828 namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: 1e559462-ec5c-4994-9d69-d4485025d7f9 kind: Event lastTimestamp: "2026-04-22T19:55:07Z" message: InferenceGraph [switch-graph-65828] is Ready metadata: creationTimestamp: "2026-04-22T19:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:55:07Z" name: switch-graph-65828.18a8c5f4a85c2d54 namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: 169f92e0-1152-42ba-a1e5-723d7b67fa41 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-65828 namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: 7f1ac0f0-4cd0-4e8f-99ba-c11f27a60b2e kind: Event lastTimestamp: "2026-04-22T19:55:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:22Z" name: switch-graph-65828.18a8c5f82212d54d namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 60660135-4d7f-4dea-a26c-a3de70cd0fe5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-65828 namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: 7f1ac0f0-4cd0-4e8f-99ba-c11f27a60b2e kind: Event lastTimestamp: "2026-04-22T19:55:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:22Z" name: switch-graph-65828.18a8c5f822769564 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 42097a26-9dc0-4b25-a3e7-d934739a5a85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-65828 namespace: kserve-ci-e2e-test resourceVersion: "25615" uid: 7f1ac0f0-4cd0-4e8f-99ba-c11f27a60b2e kind: Event lastTimestamp: "2026-04-22T19:55:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:37Z" name: switch-graph-65828.18a8c5fba10c310d namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: dd307141-9d25-4704-ab5c-3b68ab2827dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:55:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-65828 namespace: kserve-ci-e2e-test resourceVersion: "25615" uid: 7f1ac0f0-4cd0-4e8f-99ba-c11f27a60b2e kind: Event lastTimestamp: "2026-04-22T19:55:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:55:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:55:37Z" name: switch-graph-65828.18a8c5fba1503fb6 namespace: kserve-ci-e2e-test resourceVersion: "25744" uid: 89d7d250-43c4-4d20-a71e-316228efb057 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:32:46.947061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b2ca6-6684fd76f-4dcj5 to ip-10-0-143-198.ec2.internal metadata: creationTimestamp: "2026-04-22T19:32:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:32:46Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bc841d4914 namespace: kserve-ci-e2e-test resourceVersion: "14165" uid: 83922679-7ccb-480c-a235-908840e62ecd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-799c55d8b9-qjk6b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:32:47Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:32:47Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bc9d3c77a8 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: ed4c93eb-097b-4764-8e65-40d1733b3585 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ca6} kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:32:47Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" metadata: creationTimestamp: "2026-04-22T19:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:47Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bc9ea974b1 namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 4bac5bc7-5eb7-4ac0-bba2-5c81a1aaf349 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ca6} kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:32:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" in 2.897s (2.897s including waiting). Image size: 216232508 bytes.' metadata: creationTimestamp: "2026-04-22T19:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:50Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bd4b5e959d namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: bb46eece-638a-4105-9b7e-836c8f7df7af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ca6} kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:32:50Z" message: 'Created container: switch-graph-b2ca6' metadata: creationTimestamp: "2026-04-22T19:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:50Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bd4e2b851b namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: 634a35a6-331a-4d9c-ab3f-7c656d813d78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ca6} kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:32:50Z" message: Started container switch-graph-b2ca6 metadata: creationTimestamp: "2026-04-22T19:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:32:50Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bd4e9809b1 namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: db3c7199-a4a8-42aa-babe-c36243741e48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ca6} kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:33:01Z" message: Stopping container switch-graph-b2ca6 metadata: creationTimestamp: "2026-04-22T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:01Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4bfd32cda26 namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: c74eb6e7-1ac2-4706-95d6-c6ec9332a2a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ca6} kind: Pod name: switch-graph-b2ca6-6684fd76f-4dcj5 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 8e41a248-5ac8-4db6-937f-f5ac7023991e kind: Event lastTimestamp: "2026-04-22T19:33:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:27Z" name: switch-graph-b2ca6-6684fd76f-4dcj5.18a8c4c00e57077b namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 7895172d-6ba9-430a-8f6f-33f14ab5a36f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-198.ec2.internal source: component: kubelet host: ip-10-0-143-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b2ca6-6684fd76f namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: 12f1ca96-ee7f-4488-80e7-5199a86aa90d kind: Event lastTimestamp: "2026-04-22T19:32:46Z" message: 'Created pod: switch-graph-b2ca6-6684fd76f-4dcj5' metadata: creationTimestamp: "2026-04-22T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:46Z" name: switch-graph-b2ca6-6684fd76f.18a8c4bc83b17e63 namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: d13ec45b-dd17-4c35-96b9-df0dc004d2a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: c4090923-da3f-451f-adac-64f20d0bead0 kind: Event lastTimestamp: "2026-04-22T19:32:46Z" message: Scaled up replica set switch-graph-b2ca6-6684fd76f from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:32:46Z" name: switch-graph-b2ca6.18a8c4bc8304e1c7 namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: d1631a52-4db3-41b0-acc3-f91f238f50ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b707d5c8-65bb-49d1-af67-e3d86bdc0444 kind: Event lastTimestamp: "2026-04-22T19:32:47Z" message: InferenceGraph [switch-graph-b2ca6] is Ready metadata: creationTimestamp: "2026-04-22T19:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:47Z" name: switch-graph-b2ca6.18a8c4bc87777b44 namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 0fcb82eb-258d-4fd8-98c6-5f16a8e5dc27 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: b707d5c8-65bb-49d1-af67-e3d86bdc0444 kind: Event lastTimestamp: "2026-04-22T19:32:47Z" message: 'Failed to update status for InferenceGraph "switch-graph-b2ca6": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-b2ca6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:47Z" name: switch-graph-b2ca6.18a8c4bc8a576895 namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 802336b3-a974-45eb-865a-013a37a96436 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:32:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b2ca6 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: b707d5c8-65bb-49d1-af67-e3d86bdc0444 kind: Event lastTimestamp: "2026-04-22T19:32:47Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-b2ca6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:32:47Z" name: switch-graph-b2ca6.18a8c4bc8a57bb54 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 50575907-5dd9-486e-83ea-c25eb5605dc8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "32871"