--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:04.233919Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-6fb974599-27mrk to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a4c39bd9ff namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 7f670ba4-8eac-480b-9d24-5c4502533de3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a4e5a2fb2e namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: bfeda051-3c20-42fa-92e2-7001772034c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a4e7752894 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 6134f393-2c15-4b14-927f-915d30d70731 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a4ea8a6de4 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 19a5e7ea-b8be-4feb-9c2f-350901ea004e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a4eb128d78 namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 34f9c96d-f1e7-4e84-b0ef-20c7a1421e7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a5ef969887 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 6f4c3f79-1cc0-4848-9fed-a77bed5e355d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a5f33586b0 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: b897b6a8-ae3f-482d-83d8-3783854f6d1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a5f3b4a490 namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 5b69df7e-df70-4992-b5c0-9ede05db4c61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a5f3c0e99a namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 04fea98f-249f-42d7-92bd-ab5313bdef1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a5f660b138 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 2116f5bb-aafa-4bfc-b236-3062a1bffa23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a5f6dbf0ad namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 722efc47-7b4a-46e6-a59a-b1a1a68c2704 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:08:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:17Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2a667884673 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 9f964351-6967-4285-9896-50628fd16eae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2b82da4e59a namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: c766f5a3-9535-4fe0-bbc9-a6f295402aaa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2b82da56ccc namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: 817f4d6a-b317-4947-b193-a9f770c25dbf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-27mrk namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31a3fb07-714c-43e9-b93b-e15732c4e467 kind: Event lastTimestamp: "2026-09-16T19:08:32Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:32Z" name: isvc-canary-create-predictor-6fb974599-27mrk.18d5e2b943d79bc0 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 7db48f3e-b4c5-498c-90be-b6100418395b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:04.145659Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-6fb974599-5t947 to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a4be593098 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 425efe55-a608-4cc0-97ba-1e8cc2289c9f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a4e0456229 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: e6bcafc3-8acf-4592-ab55-d62ef1dc933d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a4e1e28a95 namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: f613135d-660b-4763-af9d-095e51b843c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a4e4994747 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: d88650d3-eac8-4483-a97b-5eabe0f909b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a4e50506f0 namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 89ed73d9-e769-484d-bd24-a92e5db72ef4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:08Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:08Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a5c1080598 namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: b7dda6fd-ad2e-4539-88f8-d085bebeb91b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:08Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a5c41919f2 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 46b77ac9-b4db-4d38-9173-312db67979ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:08Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a5c4a0de7b namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: db896594-ae45-4bbc-b2f4-099eac80f9b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:08Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a5c4aabfae namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: b044b6f5-38a8-4f60-ba92-77870a53104c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:08Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a5c7bba534 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: b6295b51-7cd1-45c7-a37a-b75030ff8167 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:07:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:08Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a5c827d33e namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 78280014-b122-439a-841b-8162733b10e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:08:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:36Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2a638e0b25c namespace: kserve-ci-e2e-test resourceVersion: "22176" uid: bd196448-066d-4d34-ac87-8242668fe324 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:08:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:34Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2b9b4f37757 namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: 0cb89a99-11c3-447d-a0bc-f10fc10857bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:08:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:34Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2b9b4f40a3d namespace: kserve-ci-e2e-test resourceVersion: "22129" uid: 648333d7-50ef-4a61-9a5c-fa8dd2ebbe9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-6fb974599-5t947 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 50f20665-6cb7-4b95-b94f-a649485c5050 kind: Event lastTimestamp: "2026-09-16T19:08:36Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:36Z" name: isvc-canary-create-predictor-6fb974599-5t947.18d5e2ba3f037268 namespace: kserve-ci-e2e-test resourceVersion: "22175" uid: 0578c9ae-74f0-4813-83a6-7f3657a58284 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-predictor-6fb974599 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: a4024ad9-d540-44f8-b092-1058271b9b7d kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: 'Created pod: isvc-canary-create-predictor-6fb974599-5t947' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599.18d5e2a4bd88f292 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 5d4ea698-9ff8-49fc-9551-390f5348c34c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-predictor-6fb974599 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: a4024ad9-d540-44f8-b092-1058271b9b7d kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: 'Created pod: isvc-canary-create-predictor-6fb974599-27mrk' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor-6fb974599.18d5e2a4c2da2ba7 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 542c1a80-3d55-456a-98cc-f1836c919d8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-predictor-6fb974599 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: a4024ad9-d540-44f8-b092-1058271b9b7d kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: 'Deleted pod: isvc-canary-create-predictor-6fb974599-27mrk' metadata: creationTimestamp: "2026-09-16T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create-predictor-6fb974599.18d5e2b827bad19d namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 8f8def0d-6a81-4506-a9cf-895604ec7506 reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-create-predictor namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: e2c6dc68-79c0-42d2-962c-0d2278cc59af kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Scaled up replica set isvc-canary-create-predictor-6fb974599 from 0 to 2 metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-predictor.18d5e2a4b9f4cf8a namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: 6c168cb4-fcdf-4362-862a-6de225e39912 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-create-predictor namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: e2c6dc68-79c0-42d2-962c-0d2278cc59af kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: Scaled down replica set isvc-canary-create-predictor-6fb974599 from 2 to 1 metadata: creationTimestamp: "2026-09-16T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create-predictor.18d5e2b82747fe14 namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6b9397cc-3895-47c1-8ec5-d9ccdeba221e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:04.128772Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-847844f798-cg9rs to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a4bd578587 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: a5cef2de-a102-41f0-9ef3-a47bb14fa911 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a4df5b92e7 namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: e1b92aa5-c0c0-4730-a343-ff5b39c591e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a4e120b379 namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 717d90fb-a2ca-49af-a5e6-63c995ac8c4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a4e450dc40 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: baf2e798-7d81-4933-b0f5-5d893433a594 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a4e4c5fb60 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: abdd203b-7e08-443d-bb7a-f2fe68290b33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a5f5b629db namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: b110bfaf-3c34-4d1e-a8ab-c65bbbd381b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a5f876d795 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: b6f2f8d4-c1d9-4322-8305-a3414e1bb1cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a5f8ea4ae4 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 50c53ab9-ac0e-4d6f-8a89-80ab93cb7546 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a5f8f3a650 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 7bb450c4-e68e-4e11-a5a8-4be2904b8a9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a5fc3201fb namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: b93ea16d-3e5a-41cf-80ea-c5105500478e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:07:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:09Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a5fc9dc4c8 namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 05f3039d-18a1-492e-9f3e-c32f7c022ead reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:08:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:37Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2a66d98bd6f namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 01dececf-13c6-4ebe-8efa-833d35315968 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:08:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:34Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2b9b563f684 namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: f15673f8-729a-4e72-b3f6-8ae6ef869f99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:08:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:34Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2b9b564591a namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: 73c494b0-5b34-4c6f-af95-6f2aaf26c652 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-847844f798-cg9rs namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: bf40490a-a08e-4345-8cb4-65640725cfb2 kind: Event lastTimestamp: "2026-09-16T19:08:37Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:37Z" name: isvc-canary-create-v2-predictor-847844f798-cg9rs.18d5e2ba73cb9a0f namespace: kserve-ci-e2e-test resourceVersion: "22185" uid: 4a7a980e-a664-4528-94e7-53daa856a0ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-v2-predictor-847844f798 namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: a8f1ed6c-91c5-4bb1-9423-1dacad6b39a8 kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: 'Created pod: isvc-canary-create-v2-predictor-847844f798-cg9rs' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor-847844f798.18d5e2a4bccd5eb9 namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: a05387d4-abb5-4469-9b0c-6b04372e8fac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-create-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b45466a0-502b-4414-8025-326bf2018d2a kind: Event lastTimestamp: "2026-09-16T19:07:04Z" message: Scaled up replica set isvc-canary-create-v2-predictor-847844f798 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:04Z" name: isvc-canary-create-v2-predictor.18d5e2a4b7dc3603 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 772bca88-1d6b-46dd-9bc6-1f96e1e7d02c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 21 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 1a1d6001-f16b-4367-975a-26107738de94 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create.18d5e2a4b99446b1 namespace: kserve-ci-e2e-test resourceVersion: "22055" uid: dba5e1a3-ceee-4880-9249-e981c47b2389 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 1a1d6001-f16b-4367-975a-26107738de94 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: 'Failed to update status for InferenceService "isvc-canary-create": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create.18d5e2a4ba14ee49 namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: 1ec5b588-f104-4a43-b836-d75e965df0e2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:07:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 1a1d6001-f16b-4367-975a-26107738de94 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create.18d5e2a4ba153b0b namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: bc8d64f2-b1e8-47f1-8222-fcf937a58e6f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 1a1d6001-f16b-4367-975a-26107738de94 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: InferenceService [isvc-canary-create] is Ready metadata: creationTimestamp: "2026-09-16T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:27Z" name: isvc-canary-create.18d5e2b81ff39b9f namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: 3ae03cfb-fe7b-4942-b5e4-7cc7964cc10f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:39.415381Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-649dd98487-2lzlf to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2baecddbd7d namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: 66e8bf3c-ba81-4a01-bdd3-d4cf21def628 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2baf98e6d18 namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: f95ebf99-6bc5-4439-ae2b-94f18fd60c46 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22257" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bb3274d63c namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: f1ee3033-900f-48f9-a425-319ad6f7eb38 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bb3428faac namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: d8cdb7b5-d1c2-4e8d-bbf1-43bceef177fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bb38175d75 namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: 4b56be69-e017-43ae-835d-773ffb221666 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bb388c1f33 namespace: kserve-ci-e2e-test resourceVersion: "22296" uid: f2c66105-4805-488f-a887-af305949c945 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc3062ed41 namespace: kserve-ci-e2e-test resourceVersion: "22345" uid: bbce0295-7f52-4024-b2f3-36d96b222726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc33d2396d namespace: kserve-ci-e2e-test resourceVersion: "22348" uid: 056e4a11-7122-40a6-ac81-f50a2c832e9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc345c8280 namespace: kserve-ci-e2e-test resourceVersion: "22350" uid: 7052e287-ef56-4e34-a21c-911de9dc6191 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc3466b98f namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: af36f4a5-75e9-4808-b4b7-af4f2226867f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc37452646 namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 3f145187-3bb3-4841-b61b-fcada76352b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc384f2be4 namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: 2b8f0b65-f28f-43aa-831f-843726261122 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:08:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:09:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:54Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2bc6c83e94e namespace: kserve-ci-e2e-test resourceVersion: "22869" uid: ca17cbac-fe7c-4090-8461-e6a4b0e37c78 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:10:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:02Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2ce628bc540 namespace: kserve-ci-e2e-test resourceVersion: "22947" uid: f2d08397-ecd2-4304-8487-871b3ae738cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:10:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:03Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2ce628ca586 namespace: kserve-ci-e2e-test resourceVersion: "22950" uid: 3eecfbf5-226f-44d5-beef-2a63f49d1b68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-2lzlf namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9562e8a4-b4f6-4ec1-8145-9403ca36f85d kind: Event lastTimestamp: "2026-09-16T19:10:06Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:06Z" name: isvc-canary-promote-predictor-649dd98487-2lzlf.18d5e2cf48b01611 namespace: kserve-ci-e2e-test resourceVersion: "22993" uid: 7dc7b870-f530-422e-9ff9-23d6c981e77b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:39.350828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-649dd98487-khl5g to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bae904bf59 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: d9d940e6-a8b4-431c-ac92-cfea63200c73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2baf42734c3 namespace: kserve-ci-e2e-test resourceVersion: "22261" uid: 05616d43-e407-4958-997f-ea7ae55b0543 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bb2f7bafda namespace: kserve-ci-e2e-test resourceVersion: "22284" uid: e7a2d0d9-1f77-463b-8746-3b76e1d75b94 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bb30f1eed3 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: 25fecde7-ff39-4f66-913f-92c35261240f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bb33bb354a namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: ca7e9281-e9ba-4fd8-b576-7a394a514dcb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bb3437dc61 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: e637a6ff-8fb8-41d8-8648-2bf06ddc8284 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc27e1f3d7 namespace: kserve-ci-e2e-test resourceVersion: "22336" uid: 9cfd3e4e-50cc-4a99-8fad-f9000bce4333 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc2bd70da1 namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: c546850e-f746-4b65-adc2-7e9683d50014 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc2c542463 namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: 1cb1e2e5-2f01-496c-ad06-62bd23616317 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc2c5eeb2e namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: f4ac5561-dbcc-4246-b530-0addbe00c59b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc2f57cfe2 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 484ac096-30da-411e-80a6-8e3d7ac53476 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc2fd2e760 namespace: kserve-ci-e2e-test resourceVersion: "22343" uid: 88cd4bc8-11e6-4956-9575-38302541e669 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:08:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:10:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:11Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2bc63f2e5d1 namespace: kserve-ci-e2e-test resourceVersion: "23069" uid: 058dcaac-86c1-4d7b-800c-cb8b8c1293e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:10:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:09Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2cfe1eafd91 namespace: kserve-ci-e2e-test resourceVersion: "23030" uid: bd4e8b7f-c356-4782-a106-1f306985e1bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:10:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:09Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2cfe1eb79ae namespace: kserve-ci-e2e-test resourceVersion: "23031" uid: f24b6896-0ff9-41c8-89d8-8d8fca0db88c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-649dd98487-khl5g namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 807b9513-8c4a-4fe6-b0ba-11b32bcb1648 kind: Event lastTimestamp: "2026-09-16T19:10:11Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:11Z" name: isvc-canary-promote-predictor-649dd98487-khl5g.18d5e2d06a25f8f5 namespace: kserve-ci-e2e-test resourceVersion: "23068" uid: 22154b2b-19c4-4aec-ac15-eb5988301099 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-predictor-649dd98487 namespace: kserve-ci-e2e-test resourceVersion: "22222" uid: fca58128-fd53-4429-a6dc-99c417379c28 kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: 'Created pod: isvc-canary-promote-predictor-649dd98487-khl5g' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor-649dd98487.18d5e2bae88a9340 namespace: kserve-ci-e2e-test resourceVersion: "22236" uid: d42400f6-8fd7-43d0-8daa-09857c04c941 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-predictor-649dd98487 namespace: kserve-ci-e2e-test resourceVersion: "22222" uid: fca58128-fd53-4429-a6dc-99c417379c28 kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: 'Created pod: isvc-canary-promote-predictor-649dd98487-2lzlf' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor-649dd98487.18d5e2baec001faa namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: b75a99ad-c2f1-472d-89b4-5f5d139a0a6b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-predictor-649dd98487 namespace: kserve-ci-e2e-test resourceVersion: "22940" uid: fca58128-fd53-4429-a6dc-99c417379c28 kind: Event lastTimestamp: "2026-09-16T19:10:02Z" message: 'Deleted pod: isvc-canary-promote-predictor-649dd98487-2lzlf' metadata: creationTimestamp: "2026-09-16T19:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:02Z" name: isvc-canary-promote-predictor-649dd98487.18d5e2ce62686fc0 namespace: kserve-ci-e2e-test resourceVersion: "22943" uid: 58be88e7-34dc-4b01-acb5-cfb4178d71e9 reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-predictor namespace: kserve-ci-e2e-test resourceVersion: "22221" uid: b2ee3763-3078-40d8-9ac0-f53a1455b911 kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: Scaled up replica set isvc-canary-promote-predictor-649dd98487 from 0 to 2 metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-predictor.18d5e2bae58e6a93 namespace: kserve-ci-e2e-test resourceVersion: "22224" uid: 34b65cbf-d658-4c2d-906a-32a9cf665b04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-predictor namespace: kserve-ci-e2e-test resourceVersion: "22936" uid: b2ee3763-3078-40d8-9ac0-f53a1455b911 kind: Event lastTimestamp: "2026-09-16T19:10:02Z" message: Scaled down replica set isvc-canary-promote-predictor-649dd98487 from 2 to 1 metadata: creationTimestamp: "2026-09-16T19:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:02Z" name: isvc-canary-promote-predictor.18d5e2ce61f3ea49 namespace: kserve-ci-e2e-test resourceVersion: "22941" uid: b100b82a-c845-4400-8920-fccda8741ea2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:39.339513Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bae8581436 namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 0b1734ef-a66d-4596-9e8c-50db78038f93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2baf3804694 namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: 46354297-0166-4a2b-8f92-5951c5d5abae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bb2ddb3fe3 namespace: kserve-ci-e2e-test resourceVersion: "22281" uid: 6113f548-25a8-4918-af93-9bcbfa3870bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bb2f52b57a namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 812a75f4-0f87-4400-a5b3-212a514ca52f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bb31f1a147 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: b2129675-370f-4091-a234-01f5596e6aea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:40Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bb325f1bef namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: abb740e3-a48b-4f57-9c2c-7fedd7b04931 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bc304b942b namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: b8a33bd3-dc55-47e2-b6c2-ea3b853e4303 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bc33dc9a26 namespace: kserve-ci-e2e-test resourceVersion: "22349" uid: 7bf931ab-8779-4325-8751-f1c97638f612 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bc3469e3b4 namespace: kserve-ci-e2e-test resourceVersion: "22352" uid: 2d1b3078-f568-469f-8587-b77edb231d76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bc34749e07 namespace: kserve-ci-e2e-test resourceVersion: "22353" uid: 0164eebb-db39-484e-b22b-7c1d441ed0cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bc3750f52a namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: 7b80bf1b-19ba-4204-9ec3-179cb2356368 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:08:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:44Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bc37c797fe namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: 506023b7-7783-4d3f-b82f-5f313cf79550 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:11:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:42Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2bca85ffc58 namespace: kserve-ci-e2e-test resourceVersion: "23735" uid: c0963222-8415-41c6-bc0e-dc894c5f82af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:11:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2e4e4fe1f92 namespace: kserve-ci-e2e-test resourceVersion: "23706" uid: 9dbd89ea-aca3-482f-bb12-7b9b9a79ca9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:11:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2e4e4ff6114 namespace: kserve-ci-e2e-test resourceVersion: "23708" uid: e1285c91-d8b0-414e-9ff2-8eeae744609b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 164271d6-2621-4e8c-a224-3c1e44dc24c7 kind: Event lastTimestamp: "2026-09-16T19:11:42Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:42Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk.18d5e2e5a2f743ac namespace: kserve-ci-e2e-test resourceVersion: "23734" uid: fe0ac8b3-93ef-4af8-910f-d8f6bd558dd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:10:09.492627Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T19:10:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:10:09Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2cfe5e37cef namespace: kserve-ci-e2e-test resourceVersion: "23039" uid: 692c7564-66c9-422f-919a-3bcd4adbd798 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23045" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:10Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:10:10Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d008db2d08 namespace: kserve-ci-e2e-test resourceVersion: "23049" uid: c620af0d-2a03-4964-9938-ab8584d5ed43 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:10Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d00a5f041b namespace: kserve-ci-e2e-test resourceVersion: "23051" uid: 61baba1a-0b2a-4c14-822d-1ffd514355d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:10Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d00d6e4998 namespace: kserve-ci-e2e-test resourceVersion: "23053" uid: 70fa3a33-1fcb-4c84-b667-a7a5681168e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:10Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d00dddb8cd namespace: kserve-ci-e2e-test resourceVersion: "23054" uid: af497ef3-353d-4744-90b8-ee865bdf75b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:14Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d0ff999cc4 namespace: kserve-ci-e2e-test resourceVersion: "23087" uid: e7300450-6d4c-49e8-9997-53fc5d55caf0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:14Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d102aee6d3 namespace: kserve-ci-e2e-test resourceVersion: "23089" uid: 48f17bfd-4a67-40e8-9a82-cff3d9a5bc13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:14Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d1031f8e8e namespace: kserve-ci-e2e-test resourceVersion: "23091" uid: 0cbcfbe3-a984-4dc2-ae19-c32bd7faac8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:14Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d103292750 namespace: kserve-ci-e2e-test resourceVersion: "23093" uid: 43d0a23d-0a5d-4e74-a752-968816fcadc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:14Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d106d0080b namespace: kserve-ci-e2e-test resourceVersion: "23094" uid: 4cc95635-f82a-4ff5-8601-b904afb9fde4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:10:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:14Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d1073ed947 namespace: kserve-ci-e2e-test resourceVersion: "23095" uid: 223375d2-11e4-4c8a-a5a9-7b808f31daac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:11:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:42Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2d17785c42e namespace: kserve-ci-e2e-test resourceVersion: "23731" uid: 722935df-14cd-483e-b947-5c1520cfa843 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:11:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2e4e5253ccb namespace: kserve-ci-e2e-test resourceVersion: "23707" uid: 46c43ed3-3b2b-4a88-b67d-e694e32d114e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:11:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2e4e52644b9 namespace: kserve-ci-e2e-test resourceVersion: "23709" uid: ab3ef27a-5cf1-453e-8e6b-3ae8493bfca8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 4173ef65-0d8d-4867-840a-0750921ecfeb kind: Event lastTimestamp: "2026-09-16T19:11:42Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:42Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv.18d5e2e57dc07b4f namespace: kserve-ci-e2e-test resourceVersion: "23730" uid: acf0acf8-6f44-4239-95c6-c24e6fd73d0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-v2-predictor-6f55c68fdd namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: fe28109f-f0fd-4a07-9097-a14cf72ef57a kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: 'Created pod: isvc-canary-promote-v2-predictor-6f55c68fdd-j4lpk' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd.18d5e2bae7ceb60a namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 66eda881-a447-40fc-9642-528718dbfc7f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-v2-predictor-6f55c68fdd namespace: kserve-ci-e2e-test resourceVersion: "23022" uid: fe28109f-f0fd-4a07-9097-a14cf72ef57a kind: Event lastTimestamp: "2026-09-16T19:10:09Z" message: 'Created pod: isvc-canary-promote-v2-predictor-6f55c68fdd-kz9tv' metadata: creationTimestamp: "2026-09-16T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:09Z" name: isvc-canary-promote-v2-predictor-6f55c68fdd.18d5e2cfe53a2e80 namespace: kserve-ci-e2e-test resourceVersion: "23035" uid: 1909aefb-62b4-4fac-ac63-e5c4c1b5732b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "22212" uid: ec1109e9-c351-4238-8f22-f893c14a734e kind: Event lastTimestamp: "2026-09-16T19:08:39Z" message: Scaled up replica set isvc-canary-promote-v2-predictor-6f55c68fdd from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:39Z" name: isvc-canary-promote-v2-predictor.18d5e2bae3674d16 namespace: kserve-ci-e2e-test resourceVersion: "22214" uid: b3404203-f398-4e75-b35d-e62bdcb23b64 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23016" uid: ec1109e9-c351-4238-8f22-f893c14a734e kind: Event lastTimestamp: "2026-09-16T19:10:09Z" message: Scaled up replica set isvc-canary-promote-v2-predictor-6f55c68fdd from 1 to 2 metadata: creationTimestamp: "2026-09-16T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:09Z" name: isvc-canary-promote-v2-predictor.18d5e2cfe10613ab namespace: kserve-ci-e2e-test resourceVersion: "23026" uid: 28492d13-f798-4126-a66e-96f5111bab91 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 21 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: 3b64d01f-d8b1-4160-8ef0-284d5b4b51e2 kind: Event lastTimestamp: "2026-09-16T19:10:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:03Z" name: isvc-canary-promote.18d5e2bae5305ad3 namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 69a15a58-5742-4108-ba0a-f310f2dd0105 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: 3b64d01f-d8b1-4160-8ef0-284d5b4b51e2 kind: Event lastTimestamp: "2026-09-16T19:10:03Z" message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:03Z" name: isvc-canary-promote.18d5e2bae5b0c9ba namespace: kserve-ci-e2e-test resourceVersion: "22957" uid: f275d924-936a-491b-872d-702cc836505a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:08:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: 3b64d01f-d8b1-4160-8ef0-284d5b4b51e2 kind: Event lastTimestamp: "2026-09-16T19:10:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:03Z" name: isvc-canary-promote.18d5e2bae5b11ba5 namespace: kserve-ci-e2e-test resourceVersion: "22959" uid: 7b494e62-9d44-48df-bd74-7bc787af801a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:10:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "22961" uid: 3b64d01f-d8b1-4160-8ef0-284d5b4b51e2 kind: Event lastTimestamp: "2026-09-16T19:11:32Z" message: InferenceService [isvc-canary-promote] is Ready metadata: creationTimestamp: "2026-09-16T19:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:11:32Z" name: isvc-canary-promote.18d5e2ce69749858 namespace: kserve-ci-e2e-test resourceVersion: "23639" uid: 5eec8f8c-1602-422f-bc1f-3ac00d9778cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:27.859063Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-857b7ffb9f-b75qt to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e21098b63a5e namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: ecdd39cb-a361-45ff-b74d-1fce9fae053c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e210ba6f1d89 namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: a7d71f07-39c7-4da9-8cfa-08008019c472 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e210bc11d2ac namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: ca8cf5e5-4827-49e2-b430-b1cbcd1aad90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e210be77a432 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: f4cdf843-9586-41f0-8c80-8d67bd0ebede reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e210bedcf872 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: eeaa255a-2bd8-4705-8323-ec26ac7824a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:32Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" metadata: creationTimestamp: "2026-09-16T18:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:32Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2119ef5867a namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: a401a64c-9d7a-4f5b-b667-80d942a3f5ff reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" in 16.121s (16.121s including waiting). Image size: 1580844954 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2155fde83c0 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: f45d196e-0d21-44c3-ae9b-306690a810bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2156315bd4e namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: a4a4f825-56a6-4ce2-a1a3-79fdfa793f35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e21563844f26 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: 5b4744fc-a346-4525-8e7e-c0f5b25504c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e215638f9dfc namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 6493dcf5-d096-4c38-9611-7fa0e4093bd3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.209s (3.209s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e21622d88dd3 namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 81bb4e3d-cf86-4ec7-93cd-e91337e590da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e216295d12f9 namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 6c97d6d4-67e2-40b8-95ec-9c70d2ffa4da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2162a6eb9ed namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 7151eafb-41d0-4637-adf4-451876ed7f7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:57:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:59Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2168dcc3d67 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: de77b640-3ca2-492b-b8ae-c96a7b0c2adb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:58:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:08Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2280d27f6c3 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 94efb257-52ae-4ca3-9b9e-74f57202cb45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:58:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:08Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2280d28b03e namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: d0556fa9-45d3-435f-a008-60ffffbbb3c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: f7e53ff5-93d6-4cbe-9c3f-2cc3b32472cb kind: Event lastTimestamp: "2026-09-16T18:58:09Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:09Z" name: isvc-canary-rollback-predictor-857b7ffb9f-b75qt.18d5e2284003908c namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: fb33b683-2d74-44c4-84c8-98ac772f82fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:27.973089Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-857b7ffb9f-ch7fv to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e2109f82220c namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 43a148ff-11a5-4327-9a24-3f3f91b6aaf6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e210c183d323 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 1d11678f-7e28-4272-8319-ae6b185485d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e210c2fd94b9 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 0eb1f312-9a98-4be8-9cd6-53fc27183444 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" in 3.164s (3.164s including waiting). Image size: 308251584 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:31Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e2117fa0615b namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 46cf4bb1-3e17-46a3-90fa-c603af242e77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:31Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e21182abe56d namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 0dd9cd60-d5ae-488f-a687-c484570057d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:31Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e21183278cbc namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 14535684-3dce-4375-a19d-057124871e08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" metadata: creationTimestamp: "2026-09-16T18:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:36Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e21293167a6a namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 9c99f43d-a2e1-452a-a8c8-841e411c58d6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:49Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" in 13.532s (13.532s including waiting). Image size: 1580844954 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:49Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e215b9b1586d namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 8c43f057-2151-4922-b98c-7e1eff89ccb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:49Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e215bcd3d1dd namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 4fb090f3-2d43-4cdc-9f53-85d8d90f1548 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:49Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e215bd4c8d67 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 7d671f7c-00f9-4491-a977-dccac9f105a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:49Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e215bd5adf40 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: ef74c3ca-12a6-42b7-89f3-feca6769503e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.192s (2.192s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:52Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e216400614df namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 109bb874-7e9d-4646-83d1-bd7ede79d8ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:52Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e21643604870 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 113c99ed-2741-421a-9c12-3c225afdd08b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:56:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:52Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e21643d9ad9e namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: f7c14046-a20a-46e8-9c15-1d8cf17f8719 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:59:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:49Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e2168ec31bf3 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 373c31b3-8078-444d-bafa-1ab857938dfe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:59:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:47Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e23f1eee7ff6 namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: 7ea5afa8-dac4-45d4-961c-9fe97ea9722a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:59:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:47Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e23f1eef0785 namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: 5f45f172-af5a-4c9e-8c08-f4e71c09d3f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 9b635504-47bb-4126-aed6-53ac7014c69a kind: Event lastTimestamp: "2026-09-16T18:59:49Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:49Z" name: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv.18d5e23f8963c19a namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: b234be38-f088-4643-bb63-b3b2c777f020 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:58:17.711942Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-857b7ffb9f-n29rd to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T18:58:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:58:17Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22a2c742312 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 60dba585-6257-47c0-8ad9-29baeaa846f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22a4e99f987 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: cf5d9421-efb1-4560-9fce-19145d541475 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22a50630371 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 891733ec-e17b-4d21-8537-44dee9ada3be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22a53cc054a namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 5977a9f6-2627-4e9b-a4bb-3c273882ea5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22a543a44ae namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 0915ca63-6510-4cd3-bc7c-d6d428854ff5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22b6004707c namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 9eaeb2b3-f714-4b0e-88e0-1f3b7a106296 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22b63fdd382 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 2ae62be2-2cb8-4b95-a8ba-71ce1e9e8d89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22b648687b5 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: 6a89f9c1-334d-4680-95f1-40f33b32a2a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22b64975465 namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: e9e01daf-5065-4e43-a6f9-404c5521a7cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:23Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22b67c1bb2e namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e265dfc6-22cb-4fac-89cb-2f7c4e5e2600 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:58:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:23Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22b6833de68 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 8d7f653a-3872-47f1-8df1-523182ee9002 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:59:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:33Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e22bd7ffff7c namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: b14c00c9-b41b-48a4-8b56-1d783692434c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:59:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:47Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e23f1ec0858b namespace: kserve-ci-e2e-test resourceVersion: "16735" uid: 396b880f-9877-4713-b91a-2741b433d536 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:59:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:47Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e23f1ec11f74 namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: 3bb5eda8-a33a-43c5-a433-1a9edfc97f7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e4f732d8-64d9-4693-b946-f9d08bc9023a kind: Event lastTimestamp: "2026-09-16T18:59:50Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:50Z" name: isvc-canary-rollback-predictor-857b7ffb9f-n29rd.18d5e23fde47def8 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: 81b2e44c-8d32-4306-b205-075ea1822296 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-857b7ffb9f namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 3850c4be-e451-4459-a16f-2e5a18b867c3 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: 'Created pod: isvc-canary-rollback-predictor-857b7ffb9f-b75qt' metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-predictor-857b7ffb9f.18d5e21096b3de0f namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 8bdc98c9-fd67-4d47-9198-ca3c517366f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-857b7ffb9f namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 3850c4be-e451-4459-a16f-2e5a18b867c3 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: 'Created pod: isvc-canary-rollback-predictor-857b7ffb9f-ch7fv' metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-predictor-857b7ffb9f.18d5e2109edff069 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: bbbbfa9d-4e4e-499d-a73c-44969a7f2c9c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-857b7ffb9f namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 3850c4be-e451-4459-a16f-2e5a18b867c3 kind: Event lastTimestamp: "2026-09-16T18:58:08Z" message: 'Deleted pod: isvc-canary-rollback-predictor-857b7ffb9f-b75qt' metadata: creationTimestamp: "2026-09-16T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:08Z" name: isvc-canary-rollback-predictor-857b7ffb9f.18d5e2280d047fc4 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: eba478a1-b838-4f78-b38d-70d7db970eed reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-857b7ffb9f namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 3850c4be-e451-4459-a16f-2e5a18b867c3 kind: Event lastTimestamp: "2026-09-16T18:58:17Z" message: 'Created pod: isvc-canary-rollback-predictor-857b7ffb9f-n29rd' metadata: creationTimestamp: "2026-09-16T18:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:17Z" name: isvc-canary-rollback-predictor-857b7ffb9f.18d5e22a2bbb3e91 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 2748d80d-7b73-4f93-b9c2-14e79b3cb10b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-predictor namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: ebb96b2b-70a9-4eea-9754-8323169eed39 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: Scaled up replica set isvc-canary-rollback-predictor-857b7ffb9f from 0 to 2 metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-predictor.18d5e2108ee2c7d2 namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 9e087ab7-42e7-49c3-8024-f2bd6abbae0c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-predictor namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: ebb96b2b-70a9-4eea-9754-8323169eed39 kind: Event lastTimestamp: "2026-09-16T18:58:08Z" message: Scaled down replica set isvc-canary-rollback-predictor-857b7ffb9f from 2 to 1 metadata: creationTimestamp: "2026-09-16T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:08Z" name: isvc-canary-rollback-predictor.18d5e2280c87aae5 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 6805587e-3667-44ed-9efd-bee0b589e353 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-predictor namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: ebb96b2b-70a9-4eea-9754-8323169eed39 kind: Event lastTimestamp: "2026-09-16T18:58:17Z" message: Scaled up replica set isvc-canary-rollback-predictor-857b7ffb9f from 1 to 2 metadata: creationTimestamp: "2026-09-16T18:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:17Z" name: isvc-canary-rollback-predictor.18d5e22a249510a6 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: e8c27e31-058a-413b-a88d-59fdedde9c55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:27.717481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e2109045daf2 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: fa8e8a29-2214-421f-857a-c951b0275237 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e210b235d1eb namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 5dda1f72-e820-4cfd-ad9f-a233d696816c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e210b3f83f84 namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: a418835a-6114-4789-8bce-50c161766ede reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e210b765f3a3 namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 66b57c65-f6dd-4bf8-a219-2cba5e16c6de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e210b7c7a7d9 namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: a54ae0d6-bd81-4a8e-82ad-9ac64377640e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:32Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" metadata: creationTimestamp: "2026-09-16T18:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:32Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e2119edfef16 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 872f7fda-02fa-4288-9c66-698b529ede04 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" in 16.091s (16.091s including waiting). Image size: 1580844954 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e2155dffdfec namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 11476022-1c75-424e-8755-9c77b5648df8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e215618ec11b namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 771df510-5b4f-48f8-9643-a0ff82031b06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e21562096ed8 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: c0164f13-3d71-4d17-a93d-d5df8092ea31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e2156215a776 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 1d31e18d-7dac-4e22-9e8c-07e9709663f0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.104s (3.104s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e2161b22af95 namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 2617c324-232f-4076-9453-520f7afabf40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e21626e14961 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 2b08d24e-4127-41fa-901f-fc3bead81379 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e2162826eeec namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: e319e5a9-acf4-4e59-8e78-c1f6ecaa04b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e21651cdee76 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 5f3aa533-a888-4850-987c-8f027249ac01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:58:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:17Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e22a26a62947 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: a4ccff5a-2a36-462e-8d4e-e20221fd6d20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:58:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:17Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e22a26a73b49 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: bc34abfa-9949-4c83-838d-b62e2a956e7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: d397a5ad-2723-4616-8d4e-e404da2a4c0d kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq.18d5e22a58289d28 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 5ce2f8f9-8306-4632-9fdf-003d2cb95c87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-v2-predictor-7794789c9d namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 3f369d3f-2653-4d6a-85a1-4df999861e83 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: 'Created pod: isvc-canary-rollback-v2-predictor-7794789c9d-pdjlq' metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-v2-predictor-7794789c9d.18d5e2108f9853e1 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 4754d07b-89b5-432b-8488-6af6ce173e71 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: be671c18-f6ca-45c6-bdf2-9c3a47f1ebae kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: Scaled up replica set isvc-canary-rollback-v2-predictor-7794789c9d from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-canary-rollback-v2-predictor.18d5e2108cc9c01b namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 85f1ccc1-25c8-41d9-b316-aeb78682407b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 24 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: f06ce5e3-c553-479d-b2f9-30da83cfba4c kind: Event lastTimestamp: "2026-09-16T18:58:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:18Z" name: isvc-canary-rollback.18d5e2108e80cbfd namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 7fe729e9-6ab7-419f-aaa9-c2568ed54356 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: f06ce5e3-c553-479d-b2f9-30da83cfba4c kind: Event lastTimestamp: "2026-09-16T18:59:43Z" message: InferenceService [isvc-canary-rollback] is Ready metadata: creationTimestamp: "2026-09-16T18:58:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:43Z" name: isvc-canary-rollback.18d5e22847c9c72e namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 9ba7d047-57c6-45f2-868c-ac08c6e6827c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: f06ce5e3-c553-479d-b2f9-30da83cfba4c kind: Event lastTimestamp: "2026-09-16T18:58:17Z" message: 'InferenceService [isvc-canary-rollback] is no longer Ready because of: PredictorReady, Ready, Stopped' metadata: creationTimestamp: "2026-09-16T18:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:17Z" name: isvc-canary-rollback.18d5e22a2a69072d namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 511bb235-f22a-4da8-90e9-6a7eba8c50b2 reason: InferenceServiceNotReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:52.969955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-57d5c586ff-vgfv2 to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2405a460745 namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: a0cc8e58-18c7-4df1-8c41-fdc3e82c0e4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2407c7191b8 namespace: kserve-ci-e2e-test resourceVersion: "16864" uid: e43b12d0-b1f3-4e75-8544-fbd4d4c5f0f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2407dfa1e65 namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 23e60af7-5c57-4b88-85f9-8148e104e42e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2408090998d namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: e9d6d4dd-a799-45bd-a18c-305c9060bfb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e24081087a0a namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: 2874e7f8-d1fe-400b-af06-63a18a45f994 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e24199cb37ea namespace: kserve-ci-e2e-test resourceVersion: "16933" uid: d6fb220c-a2e4-4336-93fd-8780d6e7c7d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2419d789cb0 namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 8643414b-57ce-4446-a495-80facf7e31ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2419dee2376 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: e64b5b27-e4c2-470f-8de2-2eb1f4dce337 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2419df8b8c0 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: 2cdde8dd-f1f7-4c45-b5a4-f38c1b061040 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e241a133d6fe namespace: kserve-ci-e2e-test resourceVersion: "16945" uid: fc33fa0d-b736-4411-97c1-eb1331c4c9fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e241a1bfc4bf namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: b6871a9e-8323-4585-b4d3-da36c7589a90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T19:01:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:17Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e24211f6bf0f namespace: kserve-ci-e2e-test resourceVersion: "17806" uid: 9e3b693c-b0e3-4bfb-9009-0718caf5628a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e25620814868 namespace: kserve-ci-e2e-test resourceVersion: "17890" uid: aa5ed8ed-32bc-4b6d-a041-7e3d83d9851f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vgfv2 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 41ae91b3-ae2d-4787-a8f4-cf94421c89d5 kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop-predictor-57d5c586ff-vgfv2.18d5e2562081b6b2 namespace: kserve-ci-e2e-test resourceVersion: "17891" uid: b0490521-1995-4a63-be1e-3ad67da4fe1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:52.911809Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-57d5c586ff-vj8n8 to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e24056ced87c namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: ff1eb2f9-28e1-43bf-b42e-b553504afa35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e24078783825 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 22b8096e-a5ea-476f-97f4-d5ea9b04a3a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e2407a0eda07 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: de2f8a63-e974-4f34-ab70-e115d5ac1b69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e2407c672ef3 namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: c67f17c2-cea3-4ef6-8aa5-ada4be19926c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e2407cc74d7c namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: 8d2b4f5e-fc0a-483a-9c6d-ab11c776daaa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:57Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e24180287c80 namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: c57ad7d7-5f57-4fc9-ab9b-f49e5d30663b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:57Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e2418335f332 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 2c19a8f8-92fa-4e12-b947-ed91b061213f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:57Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e241839ed096 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: a295d3a9-6ce0-44a3-8ff9-2bf352296bcd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:57Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e24183a996e9 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 2b1bb195-2fcc-4636-ab8e-262e07de6504 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e241862c5c59 namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: d3ff8858-01d6-4e1e-a627-38802885f0a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e241868d849b namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: 72bd4bb3-d823-4eee-8ed1-ac73c76b5f0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:59:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T19:01:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:35Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e241f7ed343c namespace: kserve-ci-e2e-test resourceVersion: "18004" uid: 1d721d7b-f88b-4f82-9c41-35f58db65cfb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T19:01:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:32Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e2579fcf0d7f namespace: kserve-ci-e2e-test resourceVersion: "17978" uid: 24c845dc-eaa7-4eae-8e87-96a10f27ff61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T19:01:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:32Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e2579fcfb698 namespace: kserve-ci-e2e-test resourceVersion: "17980" uid: 69951182-2f01-4d43-ad12-672b361a3608 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-57d5c586ff-vj8n8 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 47b81ac8-acf1-4aea-9b34-c42b9c3d41d6 kind: Event lastTimestamp: "2026-09-16T19:01:35Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:35Z" name: isvc-canary-stop-predictor-57d5c586ff-vj8n8.18d5e25852298792 namespace: kserve-ci-e2e-test resourceVersion: "18003" uid: cbf2501b-1923-4fd3-b1b4-3edd50a46fe8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-predictor-57d5c586ff namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 0227aed7-dd16-443b-90c0-0ceaf3520505 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: 'Created pod: isvc-canary-stop-predictor-57d5c586ff-vj8n8' metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-predictor-57d5c586ff.18d5e240564a3474 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 03d9d4f2-4ae5-4c45-851d-39879175d513 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-predictor-57d5c586ff namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 0227aed7-dd16-443b-90c0-0ceaf3520505 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: 'Created pod: isvc-canary-stop-predictor-57d5c586ff-vgfv2' metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-predictor-57d5c586ff.18d5e24059acf5eb namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: 6c89f17d-bbd9-4ca8-960b-7861c0232129 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-predictor-57d5c586ff namespace: kserve-ci-e2e-test resourceVersion: "17882" uid: 0227aed7-dd16-443b-90c0-0ceaf3520505 kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: 'Deleted pod: isvc-canary-stop-predictor-57d5c586ff-vgfv2' metadata: creationTimestamp: "2026-09-16T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop-predictor-57d5c586ff.18d5e2562047c311 namespace: kserve-ci-e2e-test resourceVersion: "17886" uid: 700e606d-cd6e-4d3f-9524-857b5e306b26 reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-stop-predictor namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: e20e0ad3-153d-4465-8ee0-285292a82903 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: Scaled up replica set isvc-canary-stop-predictor-57d5c586ff from 0 to 2 metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-predictor.18d5e24053139149 namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: c9e2d204-7322-43b2-a732-47f9c7d30380 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-stop-predictor namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: e20e0ad3-153d-4465-8ee0-285292a82903 kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: Scaled down replica set isvc-canary-stop-predictor-57d5c586ff from 2 to 1 metadata: creationTimestamp: "2026-09-16T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop-predictor.18d5e2561fb97962 namespace: kserve-ci-e2e-test resourceVersion: "17883" uid: 98e5219b-b4f3-4b63-b19a-a3c91c391663 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:52.870977Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-84ff76f474-6ggqc to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e240545fc5ac namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 15d9e848-ff51-4dda-bb31-82280e72378b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e24076b11b2f namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 24a19c3a-afd2-4976-9809-6cf7d02c201c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e240786f30c3 namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: 4feb5cc8-abf3-4818-a486-8a8c344777ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2407b512bb3 namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 20ebe694-2181-44c0-bd45-fab1a6b1e29b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:53Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2407bbd1b14 namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: c15babc9-1144-4513-84e0-8a73c8c59184 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e24199e1be3a namespace: kserve-ci-e2e-test resourceVersion: "16934" uid: 0329553d-1c3d-4751-a17a-aa03f9966aed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2419d83e57a namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 44f07ab6-be07-4336-99ec-10e95a74a20f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2419dfbf329 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: f5aa5c1d-9fd3-444e-a62d-71d19466c395 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2419e05bc54 namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: eb9e03d3-9868-466a-b78b-3f1bcb8d8bf2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e241a1405d2c namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: c5abe49c-4e65-431b-a8c1-cdca39cddfce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T18:59:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:58Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e241a1b32c29 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 14f991db-a8ac-494c-a06c-13c9ba485bda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T19:01:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:36Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e24211f4a79e namespace: kserve-ci-e2e-test resourceVersion: "18007" uid: 3c4481ee-b42a-4a63-8330-6e0e4c8d4c93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T19:01:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:32Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2579dbf1a12 namespace: kserve-ci-e2e-test resourceVersion: "17970" uid: 89f7ab11-6487-4902-9a6e-d51773279918 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T19:01:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:32Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2579dbf8f76 namespace: kserve-ci-e2e-test resourceVersion: "17972" uid: 816c80f6-8e7d-4306-a09a-bd3579d7365e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 57e7d1d5-0352-46ce-b23d-5a6ef4d84403 kind: Event lastTimestamp: "2026-09-16T19:01:36Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:36Z" name: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc.18d5e2586c3fa7a6 namespace: kserve-ci-e2e-test resourceVersion: "18006" uid: 19e098d4-a5a8-4a77-af31-ee78fa2e406a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-v2-predictor-84ff76f474 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: abe90f44-ae2f-4c60-b932-b81c80758360 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: 'Created pod: isvc-canary-stop-v2-predictor-84ff76f474-6ggqc' metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-v2-predictor-84ff76f474.18d5e24053d20e49 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 78b0a103-a656-4038-a9ad-609a949d435b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-stop-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: cdd689ac-df31-4cf6-9685-b949c2fee8c3 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: Scaled up replica set isvc-canary-stop-v2-predictor-84ff76f474 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-canary-stop-v2-predictor.18d5e24050ad0411 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 7c88abf5-cd98-4577-9ade-79eb18b27cdf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 21 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: feceafdd-ff83-4bdb-a3d6-46400811828b kind: Event lastTimestamp: "2026-09-16T19:01:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:32Z" name: isvc-canary-stop.18d5e240529a630b namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: b2f5c7f2-bd9d-4897-9205-e40f1f521448 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: feceafdd-ff83-4bdb-a3d6-46400811828b kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop.18d5e240531f3b87 namespace: kserve-ci-e2e-test resourceVersion: "17899" uid: 084365f2-32e4-44a3-ba05-41cf7c879adf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:59:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: feceafdd-ff83-4bdb-a3d6-46400811828b kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T18:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop.18d5e240531f8632 namespace: kserve-ci-e2e-test resourceVersion: "17901" uid: 2e9c51e9-d2f2-4431-bdd0-7bf3a4cc064b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "17903" uid: feceafdd-ff83-4bdb-a3d6-46400811828b kind: Event lastTimestamp: "2026-09-16T19:01:26Z" message: InferenceService [isvc-canary-stop] is Ready metadata: creationTimestamp: "2026-09-16T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:26Z" name: isvc-canary-stop.18d5e25626c6d29f namespace: kserve-ci-e2e-test resourceVersion: "17904" uid: fab63e77-aa2f-48d8-bc54-6ad0c2472928 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:03:51.719845Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-582b75-predictor-76fb588c48-467v7 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 13134712-6e89-41a9-8f8e-26fd99a92c96 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-582b75-predictor-76fb588c48-467v7 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:03:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:03:51Z" name: isvc-init-fail-582b75-predictor-76fb588c48-467v7.18d5e277f0e03fa7 namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: cd6c8f74-ef4a-40b1-a049-5c14e7598a41 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-582b75-predictor-76fb588c48-467v7 namespace: kserve-ci-e2e-test resourceVersion: "19492" uid: 13134712-6e89-41a9-8f8e-26fd99a92c96 kind: Event lastTimestamp: "2026-09-16T19:03:52Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:03:52Z" name: isvc-init-fail-582b75-predictor-76fb588c48-467v7.18d5e27812fc4234 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: 87390b7f-35e4-4e37-8504-1c035af8d312 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-582b75-predictor-76fb588c48-467v7 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 13134712-6e89-41a9-8f8e-26fd99a92c96 kind: Event lastTimestamp: "2026-09-16T19:03:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:55Z" name: isvc-init-fail-582b75-predictor-76fb588c48-467v7.18d5e2781481b5c7 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: 87c699fd-708f-4413-8306-c50f74552944 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-582b75-predictor-76fb588c48-467v7 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 13134712-6e89-41a9-8f8e-26fd99a92c96 kind: Event lastTimestamp: "2026-09-16T19:03:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:55Z" name: isvc-init-fail-582b75-predictor-76fb588c48-467v7.18d5e27817a71c32 namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: c2e64925-176e-4f0f-84a3-9368138ff4c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-582b75-predictor-76fb588c48-467v7 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 13134712-6e89-41a9-8f8e-26fd99a92c96 kind: Event lastTimestamp: "2026-09-16T19:03:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:55Z" name: isvc-init-fail-582b75-predictor-76fb588c48-467v7.18d5e2781825cfa1 namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: 054f1af1-c8ff-4179-85c4-b0202baabf29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-582b75-predictor-76fb588c48-467v7 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 13134712-6e89-41a9-8f8e-26fd99a92c96 kind: Event lastTimestamp: "2026-09-16T19:03:57Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-09-16T19:03:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:57Z" name: isvc-init-fail-582b75-predictor-76fb588c48-467v7.18d5e279434aaacb namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 2e213309-0a23-4ec4-80ca-85bd823807da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-582b75-predictor-76fb588c48 namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: ffad3f35-2252-4612-af3d-607af082aeb6 kind: Event lastTimestamp: "2026-09-16T19:03:51Z" message: 'Created pod: isvc-init-fail-582b75-predictor-76fb588c48-467v7' metadata: creationTimestamp: "2026-09-16T19:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:51Z" name: isvc-init-fail-582b75-predictor-76fb588c48.18d5e277f04b9e2e namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: ea65f7e8-1f6c-4c1c-8b7a-d7b592a42c90 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-582b75-predictor namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 94f55362-545a-43e5-9a21-1bd86365b27a kind: Event lastTimestamp: "2026-09-16T19:03:51Z" message: Scaled up replica set isvc-init-fail-582b75-predictor-76fb588c48 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:51Z" name: isvc-init-fail-582b75-predictor.18d5e277ed7bff16 namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 82aac93a-5d8d-438a-b4fe-8c1177be806b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-16T19:03:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-582b75 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: ee6a2b72-52ad-44b1-8c6d-70f4fa5c5633 kind: Event lastTimestamp: "2026-09-16T19:03:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:55Z" name: isvc-init-fail-582b75.18d5e277ed909bda namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: b0133c2d-b74f-47dc-9a4a-23ff2d04b02b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-582b75 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: ee6a2b72-52ad-44b1-8c6d-70f4fa5c5633 kind: Event lastTimestamp: "2026-09-16T19:03:51Z" message: 'Failed to update status for InferenceService "isvc-init-fail-582b75": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-582b75": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:51Z" name: isvc-init-fail-582b75.18d5e277ee0fdac5 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: dd8ba212-1675-49a3-82f2-ad2c1f93de9f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-582b75 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: ee6a2b72-52ad-44b1-8c6d-70f4fa5c5633 kind: Event lastTimestamp: "2026-09-16T19:03:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-582b75": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:51Z" name: isvc-init-fail-582b75.18d5e277ee10224e namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: 4ee6df97-088c-4109-98c3-37bc4f0eac3f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:02:33.468514Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18710" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:02:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:02:33Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e265b8bba427 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 6460f1eb-2979-4038-8492-6e655861395f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:34Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:02:34Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e265da5bb87f namespace: kserve-ci-e2e-test resourceVersion: "18725" uid: 0755f129-f7b0-4e74-ab02-ced203f9b0cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:34Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e265dc104122 namespace: kserve-ci-e2e-test resourceVersion: "18728" uid: 152901dc-bd97-4cae-85e3-d6549f5b541e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:34Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e265dee30a5a namespace: kserve-ci-e2e-test resourceVersion: "18729" uid: 5c56939b-22af-4291-8c7b-b861765383b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:34Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e265df4ea8f5 namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: 67e32a55-25f9-4346-a0fc-32daa49313be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266e472515f namespace: kserve-ci-e2e-test resourceVersion: "18769" uid: 3bf148b1-974a-4ff4-99f2-4b2aae1473ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266e792fcd6 namespace: kserve-ci-e2e-test resourceVersion: "18772" uid: eee7167d-a34d-4c8d-9654-bf287dc81880 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266e7fe5a72 namespace: kserve-ci-e2e-test resourceVersion: "18773" uid: 91a080a1-15a8-4c94-99cb-abc7487ac1c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266e808d17f namespace: kserve-ci-e2e-test resourceVersion: "18774" uid: 3c92efdb-0ea9-4a47-bcff-52719b1aae7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266eab9452e namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: c35274a6-876d-42c1-8c5a-027e24358487 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266eb27be0a namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: c8ac505d-ba32-4ede-a683-a0ef650a0ef1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:340f5bc335e7f5e59618ca4c6ecdc19e6f5d5eeba895ffabde81de29463e6007" already present on machine metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266eb32b769 namespace: kserve-ci-e2e-test resourceVersion: "18779" uid: 1acfcaca-b48a-44f3-8ffd-5b00b61b08b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266eea8854e namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: d61db65a-0d50-4eb0-8865-eba402d101cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:02:38Z" message: Started container agent metadata: creationTimestamp: "2026-09-16T19:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:38Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e266ef1cfe28 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 53a31c77-3c84-4a79-a9f7-118ad743e45c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-09-16T19:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e2672077edcc namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 96c158d4-32e5-4e31-84e9-2e855a6516a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:02:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e2675c47f977 namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 9bb00ea2-9d9d-4cba-805f-d9e5827bf750 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e27bdf90ee3f namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 4290d8f2-87cc-409c-83a7-7e02998be0bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: Stopping container agent metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e27bdf9118c2 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: ba0f7ed9-fe5b-4e02-af8c-4a3e3bb4459d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e27bdf913bad namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 0a8743ff-1753-40f6-a50f-b7e4c5b18557 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:04:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 10933a9d-e7fd-4b1a-9eda-a21f48f74639 kind: Event lastTimestamp: "2026-09-16T19:04:16Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:16Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh.18d5e27c8c89227a namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 142d75ca-212c-45e0-b84a-823c4db36df4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd namespace: kserve-ci-e2e-test resourceVersion: "18698" uid: fcbc77e4-6dfc-42cc-9808-1bc3f124f7c0 kind: Event lastTimestamp: "2026-09-16T19:02:33Z" message: 'Created pod: isvc-logger-raw-2d3bd-predictor-7fdc485bcd-mhsdh' metadata: creationTimestamp: "2026-09-16T19:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:33Z" name: isvc-logger-raw-2d3bd-predictor-7fdc485bcd.18d5e265b801602f namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: 4af67162-1b25-43c3-beb9-d3016073a925 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18696" uid: 2b8c4cdc-3091-4da1-a5a6-e2303801dde5 kind: Event lastTimestamp: "2026-09-16T19:02:33Z" message: Scaled up replica set isvc-logger-raw-2d3bd-predictor-7fdc485bcd from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:33Z" name: isvc-logger-raw-2d3bd-predictor.18d5e265b3f940b1 namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: 11343c61-9648-49cc-baa5-5dad35c7656f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:02:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18697" uid: 38dc886b-db1f-4e7d-8e04-aa99798b68f9 kind: Event lastTimestamp: "2026-09-16T19:03:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:02:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:18Z" name: isvc-logger-raw-2d3bd-predictor.18d5e269327adc76 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: e03d044d-55e0-4a1e-abfb-26a8988f4dfb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:02:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18697" uid: 38dc886b-db1f-4e7d-8e04-aa99798b68f9 kind: Event lastTimestamp: "2026-09-16T19:03:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:02:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:18Z" name: isvc-logger-raw-2d3bd-predictor.18d5e26932fc0105 namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: d2e4e018-9682-460c-97e6-254d3a2a2bce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 38dc886b-db1f-4e7d-8e04-aa99798b68f9 kind: Event lastTimestamp: "2026-09-16T19:04:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:03Z" name: isvc-logger-raw-2d3bd-predictor.18d5e273b0322628 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: bd3d1f4e-6d7b-4c99-9d8d-644cc1e7f647 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 38dc886b-db1f-4e7d-8e04-aa99798b68f9 kind: Event lastTimestamp: "2026-09-16T19:04:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:03Z" name: isvc-logger-raw-2d3bd-predictor.18d5e273b0922aa7 namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 6e602f60-599e-47ed-b9cd-edbe007bce85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:02:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2d3bd namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: 7b5f5c42-7d92-46ca-a4a6-981c4a097a97 kind: Event lastTimestamp: "2026-09-16T19:03:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:58Z" name: isvc-logger-raw-2d3bd.18d5e265b3f7a09c namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: 07b09e04-0928-4ce2-ba8a-af6217c85639 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2d3bd namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: 7b5f5c42-7d92-46ca-a4a6-981c4a097a97 kind: Event lastTimestamp: "2026-09-16T19:02:33Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-2d3bd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-2d3bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:02:33Z" name: isvc-logger-raw-2d3bd.18d5e265b49a9bf9 namespace: kserve-ci-e2e-test resourceVersion: "18703" uid: 30dcaf0f-9b1f-45ae-a451-2eae9228e20f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2d3bd namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 7b5f5c42-7d92-46ca-a4a6-981c4a097a97 kind: Event lastTimestamp: "2026-09-16T19:03:58Z" message: InferenceService [isvc-logger-raw-2d3bd] is Ready metadata: creationTimestamp: "2026-09-16T19:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:58Z" name: isvc-logger-raw-2d3bd.18d5e27997e11b73 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: adc83528-2172-4530-b4e7-8a2d9aa03b92 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:12:14.374215Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24043" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:12:14Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ecf969527e namespace: kserve-ci-e2e-test resourceVersion: "24047" uid: 1a071dd9-9840-4c25-975a-f2a2adfae28b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24053" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:14Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:12:14Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ed1b6ccc84 namespace: kserve-ci-e2e-test resourceVersion: "24061" uid: 356e49ac-8062-4196-8f06-25a550d6e318 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:14Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:14Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ed1df599f9 namespace: kserve-ci-e2e-test resourceVersion: "24063" uid: 54c7439b-ad9a-4603-8685-6dfa403a57ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:15Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ed2086db04 namespace: kserve-ci-e2e-test resourceVersion: "24064" uid: 072e9395-7c91-4591-9343-630c4d5e2789 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:15Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ed20f60c5e namespace: kserve-ci-e2e-test resourceVersion: "24065" uid: 6090d889-717d-413b-8323-57e51ce35782 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee337ff8e1 namespace: kserve-ci-e2e-test resourceVersion: "24106" uid: ac70ba7c-8785-4802-a475-7e92b1da7a57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee37c0adcf namespace: kserve-ci-e2e-test resourceVersion: "24108" uid: 37c249f4-09b1-49f6-a2b7-9db54362256c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee3839a200 namespace: kserve-ci-e2e-test resourceVersion: "24109" uid: 27d27177-dcb9-4f97-b430-a7643a886c3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee384675df namespace: kserve-ci-e2e-test resourceVersion: "24112" uid: 79d995c9-8dff-45ab-acb0-058735ae313d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee3b4bb1d2 namespace: kserve-ci-e2e-test resourceVersion: "24114" uid: 0394c04c-6bc0-4256-bfff-c4ca4161c98f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee3bbb8cea namespace: kserve-ci-e2e-test resourceVersion: "24116" uid: b03a279c-8d8e-4531-8498-3d2b40d2768f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:340f5bc335e7f5e59618ca4c6ecdc19e6f5d5eeba895ffabde81de29463e6007" already present on machine metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee3bc6cf03 namespace: kserve-ci-e2e-test resourceVersion: "24117" uid: 6620a888-b8e8-4896-bf26-c55651c9baa4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee3e6b074f namespace: kserve-ci-e2e-test resourceVersion: "24119" uid: f6606d09-cd0d-40dd-b21e-1e1d307283b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:12:19Z" message: Started container agent metadata: creationTimestamp: "2026-09-16T19:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:12:19Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2ee3ee3bc48 namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 073028eb-b00d-4d24-b05f-0416cd56034c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:14:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:14:07Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2eeab4cf6d1 namespace: kserve-ci-e2e-test resourceVersion: "24985" uid: 39596df7-aef8-4e9c-9a6c-2b459147f883 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:13:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:13:57Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e2eeab576b55 namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 87f3666c-f0e9-4176-8bb3-d3bae0f13365 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:13:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:13:49Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e303225b55f2 namespace: kserve-ci-e2e-test resourceVersion: "24798" uid: 9a1671c5-c7c3-453b-85fa-139e48e68d82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:13:49Z" message: Stopping container agent metadata: creationTimestamp: "2026-09-16T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:13:49Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e303225ba9e8 namespace: kserve-ci-e2e-test resourceVersion: "24801" uid: 8ac160ed-3c04-4c33-93ca-14ab9b5e62fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:13:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:13:49Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e303225bd3e9 namespace: kserve-ci-e2e-test resourceVersion: "24805" uid: c510c2e7-17bc-4f31-a43d-7a5ea2cb8625 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 473166d4-493a-4776-9fc5-9e2deb6a3e22 kind: Event lastTimestamp: "2026-09-16T19:14:07Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:14:07Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt.18d5e303db705291 namespace: kserve-ci-e2e-test resourceVersion: "24984" uid: 19cca1aa-9742-4e5d-8ee5-c553cb6b530b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-cipn-predictor-67b78f46c9 namespace: kserve-ci-e2e-test resourceVersion: "24031" uid: 559b7db7-7ca5-4324-8708-62a42e4f885f kind: Event lastTimestamp: "2026-09-16T19:12:14Z" message: 'Created pod: isvc-logger-raw-cipn-predictor-67b78f46c9-dhsmt' metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:14Z" name: isvc-logger-raw-cipn-predictor-67b78f46c9.18d5e2ecf8deb0cb namespace: kserve-ci-e2e-test resourceVersion: "24045" uid: 54c9d39d-a0ea-4025-8fc3-8753dc29f47c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "24028" uid: d863fe5f-4ee5-4aeb-94c6-8e2ba29592d6 kind: Event lastTimestamp: "2026-09-16T19:12:14Z" message: Scaled up replica set isvc-logger-raw-cipn-predictor-67b78f46c9 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:14Z" name: isvc-logger-raw-cipn-predictor.18d5e2ecf38b9067 namespace: kserve-ci-e2e-test resourceVersion: "24033" uid: 6000e39e-209d-46c9-9eff-aad7923e56c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "24029" uid: b724dc54-c918-434a-b599-418c4bd36067 kind: Event lastTimestamp: "2026-09-16T19:12:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:29Z" name: isvc-logger-raw-cipn-predictor.18d5e2f0726a3350 namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 6190f091-e56b-4617-a479-33c55ffa1e22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "24029" uid: b724dc54-c918-434a-b599-418c4bd36067 kind: Event lastTimestamp: "2026-09-16T19:12:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:29Z" name: isvc-logger-raw-cipn-predictor.18d5e2f072f12ae2 namespace: kserve-ci-e2e-test resourceVersion: "24195" uid: b4ca31bd-4dfb-463d-b303-bd0be0ea4173 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:12:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "24192" uid: b724dc54-c918-434a-b599-418c4bd36067 kind: Event lastTimestamp: "2026-09-16T19:13:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:13:44Z" name: isvc-logger-raw-cipn-predictor.18d5e2f3f1b70c6a namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: dc2499d5-5fff-4f4e-811c-85ffec183aae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:12:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "24192" uid: b724dc54-c918-434a-b599-418c4bd36067 kind: Event lastTimestamp: "2026-09-16T19:13:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:13:44Z" name: isvc-logger-raw-cipn-predictor.18d5e2f3f2221098 namespace: kserve-ci-e2e-test resourceVersion: "24737" uid: 75d24b03-5b8a-45b9-b61e-34a10f9aef37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:12:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "24022" uid: c1da5e24-caad-415d-a041-7035f1c7a8c8 kind: Event lastTimestamp: "2026-09-16T19:13:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:13:37Z" name: isvc-logger-raw-cipn.18d5e2ecf2fce4b2 namespace: kserve-ci-e2e-test resourceVersion: "24697" uid: 8c53cc3f-b7ff-4a3a-922a-84898a827945 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "24022" uid: c1da5e24-caad-415d-a041-7035f1c7a8c8 kind: Event lastTimestamp: "2026-09-16T19:12:14Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-cipn": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:12:14Z" name: isvc-logger-raw-cipn.18d5e2ecf3c2ae84 namespace: kserve-ci-e2e-test resourceVersion: "24034" uid: 20cba7d4-13ea-4224-85b8-3744a34568ca reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:13:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "24695" uid: c1da5e24-caad-415d-a041-7035f1c7a8c8 kind: Event lastTimestamp: "2026-09-16T19:13:37Z" message: InferenceService [isvc-logger-raw-cipn] is Ready metadata: creationTimestamp: "2026-09-16T19:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:13:37Z" name: isvc-logger-raw-cipn.18d5e30065f036ed namespace: kserve-ci-e2e-test resourceVersion: "24696" uid: 55a71f67-e013-4620-ac7c-af18a3988564 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:01:43.117243Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:01:43Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e259ff905361 namespace: kserve-ci-e2e-test resourceVersion: "18094" uid: 39f5c7bf-6189-4109-9719-6ec33762c15a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-9d9a89-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:43Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25a0c07bb4f namespace: kserve-ci-e2e-test resourceVersion: "18107" uid: d813f7e0-716c-48b5-b30a-34aaa81c45bc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18099" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:44Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:01:44Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25a45176987 namespace: kserve-ci-e2e-test resourceVersion: "18126" uid: 1f37b72b-e0b2-42fd-90ab-759fecf153f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:44Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25a46d02baa namespace: kserve-ci-e2e-test resourceVersion: "18128" uid: a1f1a8af-4e19-4692-af00-fca9e3c1e119 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:44Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25a496654f4 namespace: kserve-ci-e2e-test resourceVersion: "18129" uid: cfdd3cbd-ab03-4212-99de-5a95490d5867 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:44Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25a49d4e1f9 namespace: kserve-ci-e2e-test resourceVersion: "18130" uid: 3848c1bf-883f-46ac-955a-e2b2369c2d4c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:48Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:48Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25b351b346c namespace: kserve-ci-e2e-test resourceVersion: "18170" uid: ebbd35b3-a252-45ff-9ff3-fd05ecfe98b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:48Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25b3826a072 namespace: kserve-ci-e2e-test resourceVersion: "18172" uid: 154cbd72-0d8b-48d1-ab25-05182e3028e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:48Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25b389cec1d namespace: kserve-ci-e2e-test resourceVersion: "18173" uid: f2080b2d-13a3-4fec-827a-a2e7f135627f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:48Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25b38a90015 namespace: kserve-ci-e2e-test resourceVersion: "18174" uid: b6da3af6-c054-4d02-bdb7-e0dc16712a32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:48Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25b3b7a92f4 namespace: kserve-ci-e2e-test resourceVersion: "18177" uid: f14ccc56-7029-4dfb-9dc8-c20ccd3d490f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:01:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:01:48Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25b3bef61a0 namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: b6a97770-1737-460b-9361-56cf0f03cfca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T19:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:02:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:56Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e25bace144cf namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 42a3708c-eb89-4db4-ad93-4a861b6dfcbf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:03:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:46Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e276bd691ade namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 538895a9-8afe-4f69-b583-884357b07ef8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 0cbfecba-d5da-4c61-aa73-d780c62d87d1 kind: Event lastTimestamp: "2026-09-16T19:03:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:46Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd.18d5e276bd699540 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: f42b1d2b-2e4e-4600-bc9e-190f8bff7ed7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-9d9a89-predictor-6bdfff7bbc namespace: kserve-ci-e2e-test resourceVersion: "18079" uid: 8640d16b-08be-42ad-8c19-b504a080757c kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: 'Created pod: isvc-primary-9d9a89-predictor-6bdfff7bbc-z6vhd' metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:43Z" name: isvc-primary-9d9a89-predictor-6bdfff7bbc.18d5e259ff1634cd namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 440302df-92d7-4319-b528-93365b596f54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "18076" uid: d313fbcd-e867-48fa-8487-76ae0c6e0978 kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: Scaled up replica set isvc-primary-9d9a89-predictor-6bdfff7bbc from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:43Z" name: isvc-primary-9d9a89-predictor.18d5e259fc08611a namespace: kserve-ci-e2e-test resourceVersion: "18082" uid: 9af1520c-41cb-47ca-aaed-526488525266 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:01:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: 2eab732d-446c-4f42-9e32-1a93e100292f kind: Event lastTimestamp: "2026-09-16T19:02:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:28Z" name: isvc-primary-9d9a89-predictor.18d5e25d7aa43a0c namespace: kserve-ci-e2e-test resourceVersion: "18642" uid: c574f922-8194-4a75-9dfc-62fe046f543d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:01:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: 2eab732d-446c-4f42-9e32-1a93e100292f kind: Event lastTimestamp: "2026-09-16T19:02:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:28Z" name: isvc-primary-9d9a89-predictor.18d5e25d7b1c0ab8 namespace: kserve-ci-e2e-test resourceVersion: "18643" uid: 61c1ff9a-6383-49ae-9c0c-eff8aad4ec7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "18260" uid: 2eab732d-446c-4f42-9e32-1a93e100292f kind: Event lastTimestamp: "2026-09-16T19:03:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:43Z" name: isvc-primary-9d9a89-predictor.18d5e267f8c972fa namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: a606e5ad-6d95-4d60-9e1d-e27543eb2c6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "18260" uid: 2eab732d-446c-4f42-9e32-1a93e100292f kind: Event lastTimestamp: "2026-09-16T19:03:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:43Z" name: isvc-primary-9d9a89-predictor.18d5e267f95c7281 namespace: kserve-ci-e2e-test resourceVersion: "19349" uid: dc5a2be3-cfa6-413c-8a1e-aef3e8801dc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "18069" uid: 06a22946-62ec-44f2-aa9c-0642e4c206ee kind: Event lastTimestamp: "2026-09-16T19:03:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:04Z" name: isvc-primary-9d9a89.18d5e259fc071231 namespace: kserve-ci-e2e-test resourceVersion: "19000" uid: 26333251-2196-4fa1-9987-0ce344cd9b7d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "18069" uid: 06a22946-62ec-44f2-aa9c-0642e4c206ee kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: 'Failed to update status for InferenceService "isvc-primary-9d9a89": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-9d9a89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:43Z" name: isvc-primary-9d9a89.18d5e259fca1e029 namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: c02ce146-a4b2-45fb-a38a-724ede0b959f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "18069" uid: 06a22946-62ec-44f2-aa9c-0642e4c206ee kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-9d9a89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:43Z" name: isvc-primary-9d9a89.18d5e259fca23899 namespace: kserve-ci-e2e-test resourceVersion: "18103" uid: 026f64ab-8257-4ad5-91ed-d52675e6d3bb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 06a22946-62ec-44f2-aa9c-0642e4c206ee kind: Event lastTimestamp: "2026-09-16T19:03:04Z" message: InferenceService [isvc-primary-9d9a89] is Ready metadata: creationTimestamp: "2026-09-16T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:04Z" name: isvc-primary-9d9a89.18d5e26d067533da namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: c9439007-987e-4522-be4d-92fd6d977641 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:27.614791Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:27Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2108a26fd8f namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: 173a36a2-3324-4e8b-bc27-61f7e92bad8a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:28Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e210abf90f76 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: e12eb31d-da89-4005-9ac7-1f0afd4ac908 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:28Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" metadata: creationTimestamp: "2026-09-16T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:28Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e210ada8b8d8 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: faa74904-f66d-45a6-af0c-4627d3609fa6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" in 3.444s (3.444s including waiting). Image size: 308251584 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:31Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2117afda672 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 9bcc3607-6ef1-4c74-9042-f6f65a985230 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:31Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2117eb1c0a4 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: e5e0ad7f-1da0-42cf-b3b6-1579fc31d0de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:31Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2117f22d3b0 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 421ae8d2-5db2-4aa7-929b-5fe41919f064 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" metadata: creationTimestamp: "2026-09-16T18:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:36Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e212858cdf90 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 913d7a3c-6b64-401f-98af-2319e2657edc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" in 12.768s (12.768s including waiting). Image size: 1580844954 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2157e9e39e2 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: cb12c278-9f8f-4fab-aa53-fb0cb50a5e36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e215818c6700 namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: 1a0accef-319a-47ae-811c-fe2e414dfea3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e21581fa0be2 namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 878dc0f1-9773-4dc7-a52f-a893562b7c93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:48Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:48Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2158206c741 namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: a4e7b7f0-6297-4a90-b3b8-0cd4df0bb59e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.182s (2.182s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2160418503f namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 11c47f9f-6de6-43d8-9d7f-d6882282f138 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e21606cb040a namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 68158d20-8e19-4111-aaa0-082d2430dc71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e216073d8a38 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: e36dc213-91f8-44b6-b2cf-bc496a1cb2e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:51Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:340f5bc335e7f5e59618ca4c6ecdc19e6f5d5eeba895ffabde81de29463e6007" metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:51Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e216074bbaad namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 704d419f-70cf-4f9d-86fe-8eb04e1151a1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:340f5bc335e7f5e59618ca4c6ecdc19e6f5d5eeba895ffabde81de29463e6007" in 2.07s (2.07s including waiting). Image size: 239964771 bytes.' metadata: creationTimestamp: "2026-09-16T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:53Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e21682b78e2b namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 53da9215-24b6-416a-b36f-3990ac5aebea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:53Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-09-16T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:53Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e2168588383d namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 3144e5fd-075e-4147-a315-88df9479915e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:56:53Z" message: Started container agent metadata: creationTimestamp: "2026-09-16T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:53Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e21685f7a527 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 9def831f-6282-4400-aeab-d27d29fb6edd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T18:56:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:58:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:37Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e216f71d77ae namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 7409dbaa-6c2b-4d28-a70f-ed1cd1f9ff8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T18:56:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:58:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T18:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:37Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e216f728ac4b namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: a98dbb15-6270-4553-9081-c011c69a895c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e22b538be1fc namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 2e324a84-476d-4619-8c0b-61fa44e3a76e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Stopping container agent metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e22b538c375e namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: d234c39b-97b9-4c81-ae6a-f99ae0529920 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e22b538c92c8 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 2b26404c-5b3f-49cc-bc90-c8bb033f6644 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:58:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4de0469a-5cbb-47d0-9830-5c940b4bf320 kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj.18d5e22c276645d6 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 807b6ecc-91be-4922-b185-1cae290c127d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: c74b3249-2fe3-4f71-8641-f43254758e97 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: 'Created pod: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4-dhqlj' metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4.18d5e210898b0610 namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 7ec7e496-1f23-41ee-b16b-327e094eb022 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-b26b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: f09fe708-b712-4191-b51c-b06b9392a53d kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: Scaled up replica set isvc-raw-sklearn-batcher-b26b5-predictor-844cf8fdf4 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-raw-sklearn-batcher-b26b5-predictor.18d5e2107f98eb34 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 1e6b5997-c586-4077-9539-81655465dee5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b26b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: 6d207d56-8635-470f-a1d4-f2528c16a03f kind: Event lastTimestamp: "2026-09-16T18:57:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:27Z" name: isvc-raw-sklearn-batcher-b26b5-predictor.18d5e2140521eebb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 99b35c9f-627d-47d8-b1cf-aa0da231b000 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b26b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: 6d207d56-8635-470f-a1d4-f2528c16a03f kind: Event lastTimestamp: "2026-09-16T18:57:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:27Z" name: isvc-raw-sklearn-batcher-b26b5-predictor.18d5e21405aa3122 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 89d133b9-b74c-41af-add3-f7dfe276bbcb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b26b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 6d207d56-8635-470f-a1d4-f2528c16a03f kind: Event lastTimestamp: "2026-09-16T18:58:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:12Z" name: isvc-raw-sklearn-batcher-b26b5-predictor.18d5e22202186cce namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: e9cd1c7d-2381-4f60-a19f-38f2bdb54692 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b26b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 6d207d56-8635-470f-a1d4-f2528c16a03f kind: Event lastTimestamp: "2026-09-16T18:58:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:12Z" name: isvc-raw-sklearn-batcher-b26b5-predictor.18d5e22202a0eed8 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 17f3d51a-6471-4a2f-8444-d56db79de171 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b26b5 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 9159bc41-e6cc-460d-b8d8-b8fe8a9815d0 kind: Event lastTimestamp: "2026-09-16T18:58:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:07Z" name: isvc-raw-sklearn-batcher-b26b5.18d5e21085a877fe namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: e7c78518-09b3-4b63-b24b-e72315c9fb33 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b26b5 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 9159bc41-e6cc-460d-b8d8-b8fe8a9815d0 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-b26b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-b26b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-raw-sklearn-batcher-b26b5.18d5e210862f94ed namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 34306610-05b8-4b65-aa3c-7ee3e95ad9fd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b26b5 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 9159bc41-e6cc-460d-b8d8-b8fe8a9815d0 kind: Event lastTimestamp: "2026-09-16T18:56:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-b26b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T18:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:27Z" name: isvc-raw-sklearn-batcher-b26b5.18d5e210862fd504 namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: 6a993356-9707-4064-a6a1-ba69e07faec1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b26b5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 9159bc41-e6cc-460d-b8d8-b8fe8a9815d0 kind: Event lastTimestamp: "2026-09-16T18:58:07Z" message: InferenceService [isvc-raw-sklearn-batcher-b26b5] is Ready metadata: creationTimestamp: "2026-09-16T18:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:07Z" name: isvc-raw-sklearn-batcher-b26b5.18d5e227c0bfe751 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 43df56e4-fb4c-49ce-8429-9436fa4e24ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:03:28.300214Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: 2f40af73-f97b-46dd-b9c7-31a740451009 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-9d9a89-predictor-74798678d-rvg2t to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:03:28Z" name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t.18d5e2727cf504b3 namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: 22df331f-14cd-4540-b920-e8b9d9a804ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t namespace: kserve-ci-e2e-test resourceVersion: "19208" uid: 2f40af73-f97b-46dd-b9c7-31a740451009 kind: Event lastTimestamp: "2026-09-16T19:03:28Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:03:28Z" name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t.18d5e2729f0aa9e7 namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: f0759e78-413c-4ecc-817e-c15970a33a16 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 2f40af73-f97b-46dd-b9c7-31a740451009 kind: Event lastTimestamp: "2026-09-16T19:03:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:35Z" name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t.18d5e272a0989f1a namespace: kserve-ci-e2e-test resourceVersion: "19281" uid: 637d9eb3-cb42-40fb-9e80-949b7e1e9212 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 2f40af73-f97b-46dd-b9c7-31a740451009 kind: Event lastTimestamp: "2026-09-16T19:03:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:35Z" name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t.18d5e272a323890b namespace: kserve-ci-e2e-test resourceVersion: "19283" uid: 8c0faf38-a5f7-4328-a474-ebb7a471a4d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 2f40af73-f97b-46dd-b9c7-31a740451009 kind: Event lastTimestamp: "2026-09-16T19:03:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:35Z" name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t.18d5e272a390679d namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: fd307651-fee2-49ad-b8f3-9550aea23e00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 2f40af73-f97b-46dd-b9c7-31a740451009 kind: Event lastTimestamp: "2026-09-16T19:03:41Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-9d9a89-predictor-74798678d-rvg2t_kserve-ci-e2e-test(2f40af73-f97b-46dd-b9c7-31a740451009) metadata: creationTimestamp: "2026-09-16T19:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:03:41Z" name: isvc-secondary-9d9a89-predictor-74798678d-rvg2t.18d5e2758bb4cc40 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: 9349a1fa-de97-45e4-8d24-642427226e03 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-9d9a89-predictor-74798678d namespace: kserve-ci-e2e-test resourceVersion: "19189" uid: 0d495264-9602-4029-9fda-d26e161bfc9d kind: Event lastTimestamp: "2026-09-16T19:03:28Z" message: 'Created pod: isvc-secondary-9d9a89-predictor-74798678d-rvg2t' metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:28Z" name: isvc-secondary-9d9a89-predictor-74798678d.18d5e2727c5e19d8 namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 2079ef02-0cd5-4574-970d-5376bdde3720 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "19187" uid: be974e95-c60a-4aac-b869-d94780c77c59 kind: Event lastTimestamp: "2026-09-16T19:03:28Z" message: Scaled up replica set isvc-secondary-9d9a89-predictor-74798678d from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:28Z" name: isvc-secondary-9d9a89-predictor.18d5e272798ac82e namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: c5b2f718-8211-43a6-abcd-6f985b14b51c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "19188" uid: 5eadb826-c2db-4f62-a1f1-838a4bdb73d3 kind: Event lastTimestamp: "2026-09-16T19:03:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:03:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:43Z" name: isvc-secondary-9d9a89-predictor.18d5e275f7f43d72 namespace: kserve-ci-e2e-test resourceVersion: "19350" uid: 3a8d7aff-a6e0-40fa-a174-70af8582f014 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-9d9a89-predictor namespace: kserve-ci-e2e-test resourceVersion: "19188" uid: 5eadb826-c2db-4f62-a1f1-838a4bdb73d3 kind: Event lastTimestamp: "2026-09-16T19:03:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:03:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:03:43Z" name: isvc-secondary-9d9a89-predictor.18d5e275f85f18e2 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: 264eb876-d105-43db-9abc-4b18e8a17bfa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 8548d1cb-63b3-4ac6-8bad-b73baf663820 kind: Event lastTimestamp: "2026-09-16T19:03:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:41Z" name: isvc-secondary-9d9a89.18d5e272798af6a8 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 9c0909c1-ff46-40ee-ab95-9f77c66a81d8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 8548d1cb-63b3-4ac6-8bad-b73baf663820 kind: Event lastTimestamp: "2026-09-16T19:03:28Z" message: 'Failed to update status for InferenceService "isvc-secondary-9d9a89": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-9d9a89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:28Z" name: isvc-secondary-9d9a89.18d5e2727a20db17 namespace: kserve-ci-e2e-test resourceVersion: "19195" uid: d6d4ea43-3fa9-45a9-97ce-0da40afc4b92 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:03:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-9d9a89 namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 8548d1cb-63b3-4ac6-8bad-b73baf663820 kind: Event lastTimestamp: "2026-09-16T19:03:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-9d9a89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:03:28Z" name: isvc-secondary-9d9a89.18d5e2727a2119ff namespace: kserve-ci-e2e-test resourceVersion: "19196" uid: 614fe02b-967c-4ab7-986e-9bedaf77801e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:58:22.861242Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:58:22Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22b5f6026ac namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: c9b7eada-a0db-482f-a71d-e81fb681f83e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-f7202-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:22Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22b66f0c61c namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 8450c579-5542-4405-bbec-476059ae4cf8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:58:24Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22ba5a67db5 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: e8b70f74-4a31-4365-8991-5f4fc714584b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22ba72d1ee4 namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: f6ed3b60-9dfe-48ff-8c57-a986725329e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22baa2103de namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 611b545c-f590-4463-acb4-85549d6ead8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22baaa10867 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 14bcee34-fc96-4d07-93e5-391aff6423da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22cc6aa2541 namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: a3da3824-d942-4ad9-aaf0-6e8dfebc948b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22cca5147db namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 3718fd8a-4860-4d0b-bebe-933d460fda11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22ccadbffca namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: a0fda1cc-fe1e-493d-9191-fd54a24aaf71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22ccae654f2 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: b5dfa1dd-45b7-4587-9928-79c7397801f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:29Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22cce4403d6 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: bfa1ba3e-1984-4f39-8420-00a877f5d0a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T18:58:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:29Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22ccece8260 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: e3c5faa8-cbc8-4b4c-b8ac-dff82053f32b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T19:00:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:15Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e22d02bf385f namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 75348f56-fc3e-43a5-815d-201752627e0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T19:00:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:12Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e24503e7ffb0 namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: 296cb32b-b7c8-464b-b519-7b325822eebf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T19:00:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e24503e88dc6 namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: b9704833-6f4d-4540-95f0-e0d96e0620f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 76d94e8a-c5bf-4f5a-b2a6-a2b91fc88dbd kind: Event lastTimestamp: "2026-09-16T19:00:15Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:00:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:15Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c.18d5e245b10aef30 namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: d9a1fff5-ff4e-425b-8210-8dc2c8b1ffa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4 namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 04fe65da-b5a4-4276-bb97-df6423063939 kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: 'Created pod: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4-rs22c' metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:22Z" name: isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4.18d5e22b5e91c1ea namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: b19816ec-2c7a-4b6b-94d4-e5a58be8d6b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 0f4ad07d-97d1-445c-be64-c4064bcfa111 kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Scaled up replica set isvc-sklearn-graph-raw-f7202-predictor-55bcf87df4 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:22Z" name: isvc-sklearn-graph-raw-f7202-predictor.18d5e22b59796197 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 7e1e2100-acf6-4b9b-a6ce-89f994091a06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: b704f7ca-252a-4ed9-99e4-1cfa05d16625 kind: Event lastTimestamp: "2026-09-16T18:58:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:52Z" name: isvc-sklearn-graph-raw-f7202-predictor.18d5e22ed96cab2d namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 755b50f0-768a-4da1-afa4-e907ce1e3be4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: b704f7ca-252a-4ed9-99e4-1cfa05d16625 kind: Event lastTimestamp: "2026-09-16T18:58:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:52Z" name: isvc-sklearn-graph-raw-f7202-predictor.18d5e22ed9f049a2 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: f06a949d-4281-4ead-bb6c-c936133b3a83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:59:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: b704f7ca-252a-4ed9-99e4-1cfa05d16625 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-sklearn-graph-raw-f7202-predictor.18d5e235d7d3a991 namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 6c7c5935-d6e0-4511-877d-57133d7dec11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:59:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: b704f7ca-252a-4ed9-99e4-1cfa05d16625 kind: Event lastTimestamp: "2026-09-16T18:59:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:52Z" name: isvc-sklearn-graph-raw-f7202-predictor.18d5e235d83c5dd7 namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: d87a6dd4-8ebb-4979-94f0-360376caf668 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 93ace822-d068-407f-9e4f-d46155f7ba7c kind: Event lastTimestamp: "2026-09-16T18:59:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:46Z" name: isvc-sklearn-graph-raw-f7202.18d5e22b595ee79d namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: bcae4a79-9c05-477b-af86-5b51a00fc519 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 93ace822-d068-407f-9e4f-d46155f7ba7c kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-f7202": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-f7202": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:22Z" name: isvc-sklearn-graph-raw-f7202.18d5e22b5a88c620 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 7eb4b350-80f2-4614-a17c-3b09626dda5b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: 93ace822-d068-407f-9e4f-d46155f7ba7c kind: Event lastTimestamp: "2026-09-16T18:59:45Z" message: InferenceService [isvc-sklearn-graph-raw-f7202] is Ready metadata: creationTimestamp: "2026-09-16T18:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:45Z" name: isvc-sklearn-graph-raw-f7202.18d5e23eba82e982 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: d0abb9cb-bd60-4187-8d9b-6c329cb31d90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:00:13.101931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17195" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2450a3bdbc0 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: 40d81431-7f2f-46c6-b308-81ec4ae3a103 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17217" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2452ca00e67 namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: db66f4ce-16ff-4c4b-870a-1fe5d0fe5f61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2452e37c720 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: e9163099-1a3f-4418-8cfb-a34d1dba5c19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e24530d42adb namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: 95149494-c74c-4361-8289-b4d2f02b4882 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e245317742fc namespace: kserve-ci-e2e-test resourceVersion: "17255" uid: 99aa9d8a-aa50-4b3a-af38-894489a68c36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e246488be021 namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 2680c2af-cc95-4044-ae16-63c463bf1c80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2464c23eb47 namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 1a3b39ab-14c2-4af3-90e7-08c3e06ecaf9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2464c955fa0 namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 3520f5c7-7d7b-4ebb-a7cd-80a1e17aafa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2464ca02e01 namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: fd94fc7d-6bd9-4ddf-a32c-8211fce4de15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2464f7f5250 namespace: kserve-ci-e2e-test resourceVersion: "17322" uid: b1f25035-7b44-45b1-afa2-3655fbae4564 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2464ff530b7 namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: 302ad3c2-280c-4f29-8d15-37443ab978d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:00:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e246850e74f2 namespace: kserve-ci-e2e-test resourceVersion: "18524" uid: 70864b30-f7cf-4717-afdb-2f0b6c44c5c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2610ac1e87c namespace: kserve-ci-e2e-test resourceVersion: "18463" uid: 3f02be70-f74c-40b2-bf1b-143e97d60523 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e2610ac27099 namespace: kserve-ci-e2e-test resourceVersion: "18464" uid: 88c0327e-24ec-46e4-908e-7a7e78d1a07b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 13ef0593-7912-4e59-a2cc-8a9f936fa0b9 kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht.18d5e26187734a14 namespace: kserve-ci-e2e-test resourceVersion: "18523" uid: 71c7ed19-af6b-407b-a627-2670acc128d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: a26a0af0-5c24-4566-ab71-127ee4459a0a kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d-j7fht' metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d.18d5e2450978af96 namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: 201fb1ca-1ff4-4bdb-8359-c923c5e931f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 724a88f8-8b85-47af-ab03-b2aedbd80ea8 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-9ebe2-predictor-79d95b6d9d from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor.18d5e24506a3ed19 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: 690fc914-b8bd-4b86-9efc-86aa2161aa35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: ec55c9f9-bb2f-4c25-9e1e-d05ca439ae42 kind: Event lastTimestamp: "2026-09-16T19:00:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:58Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor.18d5e248852a1141 namespace: kserve-ci-e2e-test resourceVersion: "17650" uid: 96c7daa1-00ce-4fc5-ba9c-9e007b318315 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: ec55c9f9-bb2f-4c25-9e1e-d05ca439ae42 kind: Event lastTimestamp: "2026-09-16T19:00:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:58Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor.18d5e248858ea9e1 namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: d9ab2a44-d911-4854-978a-0831d77105f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:01:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: ec55c9f9-bb2f-4c25-9e1e-d05ca439ae42 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:01:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor.18d5e2530341f12b namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: 872523cc-ff6f-4a9a-a136-14d83d519808 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:01:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: ec55c9f9-bb2f-4c25-9e1e-d05ca439ae42 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:01:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2-predictor.18d5e25303b4d879 namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: 1f886d42-e709-433a-b8e9-ff78132fbc13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 34748945-7b46-449b-8507-16f2e24cb43c kind: Event lastTimestamp: "2026-09-16T19:01:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:45Z" name: isvc-sklearn-graph-raw-hpa-9ebe2.18d5e24506a5c594 namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: dfa05f08-27e8-4e0a-955f-481d10e42265 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 34748945-7b46-449b-8507-16f2e24cb43c kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-9ebe2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-9ebe2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:13Z" name: isvc-sklearn-graph-raw-hpa-9ebe2.18d5e245077ec518 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 00204dc9-77c3-40ff-a6df-0a4124b59955 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "18151" uid: 34748945-7b46-449b-8507-16f2e24cb43c kind: Event lastTimestamp: "2026-09-16T19:01:45Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-9ebe2] is Ready metadata: creationTimestamp: "2026-09-16T19:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:45Z" name: isvc-sklearn-graph-raw-hpa-9ebe2.18d5e25a912987d9 namespace: kserve-ci-e2e-test resourceVersion: "18152" uid: 86ca2816-7ba6-4f78-ad0c-903b64a8a10d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:04:08.751112Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:04:08Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27be804c9d6 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 7baebf5c-b53d-4e59-96d1-d5acdfa2a0cd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:09Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:04:09Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27c0a2abe5e namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: f4a0dc5d-4b0c-4dc5-993b-e07f026f5d94 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:09Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27c0ba81c72 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: d6ef8a8f-19b0-4353-8a03-38ce07f9b9c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:09Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27c0e83d470 namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: 1b72b7a6-c2cd-4bf0-bc15-98213aea344a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:09Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27c0f0f0fee namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: decedae5-d1fa-43b8-b343-49db3d87d161 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:13Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:13Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d0181322d namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: ce1625c1-0c93-4fc5-9727-19157be752a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:13Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d0509c4f4 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 109dfbe5-ffe8-49bb-a5ad-4ef089e590e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:13Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d058c0cac namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: b9c2b2db-cfad-4ea6-8d48-bafb15be4197 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:13Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d0597a6b7 namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 633a75ba-fec9-4e3f-8eee-1c143684c673 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:13Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d08871e72 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: a3ba22ec-9284-47b3-a980-0d58aa3054d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:04:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:13Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d09078c4f namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: b75db399-bb8e-4ef7-889f-66c9e5518888 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-09-16T19:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:07:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:05Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e27d3d8d0f54 namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: a97fe9e5-2ac9-43dc-bf73-ddc611b11b1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:06:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:58Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e2a384278a8c namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: ff3a35d1-305e-4475-a64d-93ff84a27fb6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:06:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:58Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e2a384283d78 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 30d4ed75-ddd7-451f-b10d-2052401b99e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41f9d2db-6097-4ea6-9de7-70304a5ee0a5 kind: Event lastTimestamp: "2026-09-16T19:07:05Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:05Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2.18d5e2a3e429b6e4 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: c7580c68-cc13-48af-a4c3-d69e57119a24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 2680f09b-5d9a-47fe-9d41-4cec6e386157 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: 'Created pod: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968-5xhh2' metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:08Z" name: isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968.18d5e27be7962d60 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: b3cbac05-12ce-42fa-a478-3a6ba23419a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-1ed38-predictor namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: c00160fc-3d1a-410a-84dd-c235d98d9ea1 kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: Scaled up replica set isvc-sklearn-scale-raw-1ed38-predictor-69ffcf968 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:08Z" name: isvc-sklearn-scale-raw-1ed38-predictor.18d5e27be4e8ab7b namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 23c88360-e6b5-4097-ad6d-16e4ab88fc86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1ed38-predictor namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 803516ad-63d7-45ba-9f4f-07c59efa44df kind: Event lastTimestamp: "2026-09-16T19:04:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:38Z" name: isvc-sklearn-scale-raw-1ed38-predictor.18d5e27f6385dbbc namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 43d148a3-3f2b-40c0-9da2-b0647ce9bbeb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1ed38-predictor namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 803516ad-63d7-45ba-9f4f-07c59efa44df kind: Event lastTimestamp: "2026-09-16T19:04:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:38Z" name: isvc-sklearn-scale-raw-1ed38-predictor.18d5e27f63f37c63 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 73188d52-6267-42fe-a67d-73e8a6baf7d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T19:04:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1ed38-predictor namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: 803516ad-63d7-45ba-9f4f-07c59efa44df kind: Event lastTimestamp: "2026-09-16T19:06:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:53Z" name: isvc-sklearn-scale-raw-1ed38-predictor.18d5e28661684d73 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 49544801-a2df-4579-a7b5-3aacb828d319 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T19:04:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1ed38-predictor namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: 803516ad-63d7-45ba-9f4f-07c59efa44df kind: Event lastTimestamp: "2026-09-16T19:06:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:53Z" name: isvc-sklearn-scale-raw-1ed38-predictor.18d5e28661da51df namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 12d975bc-842c-4d8c-b534-7b3eac19d64e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1ed38 namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 9040409b-09cc-4386-8543-9ee673a17842 kind: Event lastTimestamp: "2026-09-16T19:06:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:55Z" name: isvc-sklearn-scale-raw-1ed38.18d5e27be4c13646 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: b6c1b387-49bd-437b-868d-241f4c50e188 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1ed38 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9040409b-09cc-4386-8543-9ee673a17842 kind: Event lastTimestamp: "2026-09-16T19:06:55Z" message: InferenceService [isvc-sklearn-scale-raw-1ed38] is Ready metadata: creationTimestamp: "2026-09-16T19:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:55Z" name: isvc-sklearn-scale-raw-1ed38.18d5e2a2bd060646 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: dce1114f-4c06-4cba-a770-d35a576c90ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:58:22.986232Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:58:22Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22b66d369bd namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 1c9d04b2-ea08-4ee0-a3c9-04ca59bd6510 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-f7202-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:23Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22b7670e8d1 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: d6faf42d-f8d3-4ba7-a006-d387967be6f9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:58:24Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22bac61ffd7 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 134b8394-8c21-455b-9345-5170f034ea3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22bae132293 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 9800bcf2-1fab-4799-a752-ddac5ce27a10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22bb11767e5 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 1be0c829-71a5-49c0-90e3-75b9e859ab93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22bb1744cbd namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: bb799082-c11a-4772-b710-fb89ec87551f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e22cb290a711 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 3aca1370-a243-4ca5-9585-ece150046458 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:46Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.69s (17.69s including waiting). Image size: 1332365570 bytes.' metadata: creationTimestamp: "2026-09-16T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:46Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e230d0fe3ffc namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 6ff40ae6-22f9-4617-bd04-d8a889a146bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:46Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e230d4bb2688 namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 3d066a72-fee9-492c-8f4e-ed6626808683 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:46Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e230d5469775 namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 92b50055-3aaf-42ba-84a0-2842c7049747 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:46Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e230d550d221 namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: b6ef6106-cc59-4ed8-9b8d-000e657a56a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:46Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e230d8967209 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 92cf902c-d2f3-495c-8466-e578a3d353a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T18:58:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:46Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e230d90637d8 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 69911449-513a-4000-8d39-db70fe0fb1b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e231240f2481 namespace: kserve-ci-e2e-test resourceVersion: "17249" uid: cf17f195-2b92-474a-99d8-aa42b7506a6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T19:00:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:12Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e2450172409d namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 53753231-709d-40b8-8186-0f1f10cfd041 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T19:00:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:12Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e2450172b100 namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: a4d332ab-00dd-480d-818e-28c803048b41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aff5faf1-bf8e-4f25-8ff8-c0c43df217d1 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv.18d5e2452a372a75 namespace: kserve-ci-e2e-test resourceVersion: "17248" uid: 81d29f42-b8fc-48c6-8c35-dc5bf20254c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 1f3fd23c-bab5-47a3-8742-e2fad943e51b kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: 'Created pod: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc-qkzbv' metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:22Z" name: isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc.18d5e22b66455d1c namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 12664762-b02f-4d22-b115-d89731da6f3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 569bc0d1-fa5a-4bad-8afb-d1493f796ce3 kind: Event lastTimestamp: "2026-09-16T18:58:22Z" message: Scaled up replica set isvc-xgboost-graph-raw-f7202-predictor-855ccc46fc from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:22Z" name: isvc-xgboost-graph-raw-f7202-predictor.18d5e22b61f93b3f namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 5be44843-3ba0-42fe-8e92-5823ba6aa699 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5b06d643-56da-4add-b2fe-269ea3ef6e9d kind: Event lastTimestamp: "2026-09-16T18:58:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:52Z" name: isvc-xgboost-graph-raw-f7202-predictor.18d5e22ee050f050 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 49959238-600f-4136-a7bc-34feefd82a55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5b06d643-56da-4add-b2fe-269ea3ef6e9d kind: Event lastTimestamp: "2026-09-16T18:58:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:52Z" name: isvc-xgboost-graph-raw-f7202-predictor.18d5e22ee0cf5b38 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 116973ac-8cf4-48f6-ad69-8258a3c6fcc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:59:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 5b06d643-56da-4add-b2fe-269ea3ef6e9d kind: Event lastTimestamp: "2026-09-16T19:00:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:07Z" name: isvc-xgboost-graph-raw-f7202-predictor.18d5e235de3fb16e namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: 20c9dd65-a168-4d3a-9b6e-733d6b5a9546 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:59:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f7202-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 5b06d643-56da-4add-b2fe-269ea3ef6e9d kind: Event lastTimestamp: "2026-09-16T19:00:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:07Z" name: isvc-xgboost-graph-raw-f7202-predictor.18d5e235deb33a80 namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 5e9e4fdf-1534-4681-bc37-10e38eb77427 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:58:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: dc36e81c-67c0-45f5-ab20-1da372d44cb8 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:53Z" name: isvc-xgboost-graph-raw-f7202.18d5e22b61c7ec99 namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: 196f9f86-3cec-4c57-9950-c64c9cf3a878 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: dc36e81c-67c0-45f5-ab20-1da372d44cb8 kind: Event lastTimestamp: "2026-09-16T18:59:53Z" message: InferenceService [isvc-xgboost-graph-raw-f7202] is Ready metadata: creationTimestamp: "2026-09-16T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:53Z" name: isvc-xgboost-graph-raw-f7202.18d5e24087be3f31 namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 8b15ea6e-8a42-4d4b-94e9-19b738de7cd2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:00:13.228600Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:00:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24511c87516 namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: de249298-503b-48e7-a381-90ab4db0e3c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-9ebe2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e2451eaaa219 namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: f5f268cd-ca82-4755-b276-038d0ca0c96a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17232" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:14Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:00:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:00:14Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24557a8fa11 namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: b9bf4b9b-68b5-499e-b2c3-4d0e51b718ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:14Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:00:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:14Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24559606249 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: e3b1d9bd-6edb-4656-b302-998345b643bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:00:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:14Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e2455ce97e02 namespace: kserve-ci-e2e-test resourceVersion: "17266" uid: 88820803-13ec-4fbd-a2dc-78481e736ae5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:00:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:14Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e2455d5abccd namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: 944cb55c-13ea-4846-9ea0-654c41655803 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:18Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-16T19:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:18Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e246487b4ca2 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: 7bf0468c-0338-418c-b3df-b40a49ffeed6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:36Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.214s (18.214s including waiting). Image size: 1332365570 bytes.' metadata: creationTimestamp: "2026-09-16T19:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:36Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24a86262e33 namespace: kserve-ci-e2e-test resourceVersion: "17476" uid: 027af51b-edfd-4b29-998c-ae3193b97934 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:36Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24a895d1761 namespace: kserve-ci-e2e-test resourceVersion: "17478" uid: 010de677-883a-4fc7-871e-74ffbe8c7c67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:36Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24a89ca152c namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 0ae6492d-06cc-4b2a-b789-b45c2512ef24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:36Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24a89d3ec82 namespace: kserve-ci-e2e-test resourceVersion: "17480" uid: 757f1431-6bcd-439a-be79-c34c230ac413 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:36Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24a8d47eb34 namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 25e1925a-30f2-4031-a2ec-84f90f53009a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:00:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:36Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24a8dbb25a1 namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: 6db511b5-397c-4f60-a00b-af42c628a0ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T19:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e24abc049fe6 namespace: kserve-ci-e2e-test resourceVersion: "18501" uid: 320b7961-e3ab-4cf3-9072-9d4371158841 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e2610fa60287 namespace: kserve-ci-e2e-test resourceVersion: "18492" uid: a0c00ff6-f64c-4a39-96e6-c71c7701f8c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e2610fa666c8 namespace: kserve-ci-e2e-test resourceVersion: "18493" uid: 0ea35dd1-db76-4744-a570-f6b55b249405 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 3519cdf2-2758-4e15-a22d-dd7488c0c456 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt.18d5e2611652f5c9 namespace: kserve-ci-e2e-test resourceVersion: "18500" uid: c53cd386-02c0-40ef-94fa-ddbcd846e99d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969 namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: f7554ae5-0621-44f4-8410-6a0f1bbcac3b kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969-8fmxt' metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969.18d5e2451118a8e4 namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: fbef5575-9448-47cb-8b16-b4c9204e1da6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 9b31ffd8-c69f-4cee-8fda-de7e3286d24d kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-9ebe2-predictor-c767cd969 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor.18d5e2450db79bdf namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: a8340d85-8232-4aba-ba94-f183dd0734d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:00:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: b768940d-beab-4eb0-acda-235abf797e14 kind: Event lastTimestamp: "2026-09-16T19:01:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor.18d5e2488c4a7d5b namespace: kserve-ci-e2e-test resourceVersion: "17766" uid: d5318905-fba2-42b7-b87a-6b82945715ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:00:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: b768940d-beab-4eb0-acda-235abf797e14 kind: Event lastTimestamp: "2026-09-16T19:01:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor.18d5e2488cb5fe32 namespace: kserve-ci-e2e-test resourceVersion: "17767" uid: b6726bd9-7482-4761-96e0-d343b9bc3f11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:01:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17406" uid: b768940d-beab-4eb0-acda-235abf797e14 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor.18d5e25687a946cb namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 485f86df-2c21-43a0-bb81-9305e80959a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:01:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17406" uid: b768940d-beab-4eb0-acda-235abf797e14 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:13Z" name: isvc-xgboost-graph-raw-hpa-9ebe2-predictor.18d5e2568822c77a namespace: kserve-ci-e2e-test resourceVersion: "18430" uid: 165cdfc6-ca88-4d06-b191-ad272398011d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: f88e78ae-0a0f-472b-8e1c-3977c41e5e20 kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:43Z" name: isvc-xgboost-graph-raw-hpa-9ebe2.18d5e2450dce2ecc namespace: kserve-ci-e2e-test resourceVersion: "18120" uid: f4e5cea3-89f1-4c70-8259-12be9d5867d6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:01:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: f88e78ae-0a0f-472b-8e1c-3977c41e5e20 kind: Event lastTimestamp: "2026-09-16T19:01:43Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-9ebe2] is Ready metadata: creationTimestamp: "2026-09-16T19:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:01:43Z" name: isvc-xgboost-graph-raw-hpa-9ebe2.18d5e25a203fd6d4 namespace: kserve-ci-e2e-test resourceVersion: "18119" uid: fc27759b-8118-42f8-b13d-626384e13423 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:02:13.396591Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:02:13Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e2610c5a7490 namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: c780b76b-7a6d-41e8-b31f-cd81604f8a2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-2d3bd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e26114cbe9f6 namespace: kserve-ci-e2e-test resourceVersion: "18498" uid: 9a253c86-0ed4-4c5f-8c38-9e3db8b91700 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:14Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18488" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:14Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:02:14Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e26151ec5f09 namespace: kserve-ci-e2e-test resourceVersion: "18511" uid: 84eb9369-4503-4187-afea-423ab943eaa7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:14Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-09-16T19:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:14Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e261539079e7 namespace: kserve-ci-e2e-test resourceVersion: "18513" uid: 9f0bb5bf-3b31-44fd-ba12-08a1436d3c0e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.172s (1.172s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e261996d5073 namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: 1215ca02-52a2-40e1-90df-30f269250e13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e2619c164f3e namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: 03fe8e3b-b25d-4565-9148-f8e1f25f79f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e2619c87cf2d namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: 5c1777de-af04-42ba-a156-964c1c7560e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e2619c924a91 namespace: kserve-ci-e2e-test resourceVersion: "18530" uid: 11334530-489d-40d6-8820-425531e165d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e2619f9151ff namespace: kserve-ci-e2e-test resourceVersion: "18531" uid: 11e15550-7a11-4961-b257-d5ae14849e29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:02:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:15Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e2619fffda91 namespace: kserve-ci-e2e-test resourceVersion: "18532" uid: 134c3a74-07e3-4548-8862-19afe16f3eb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e27be2175063 namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: 172bb44c-c419-421c-a462-180b3838a6cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5 namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: b2e9d1eb-3932-44a0-9bbb-f023649c868d kind: Event lastTimestamp: "2026-09-16T19:04:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:08Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5.18d5e27be217cc0a namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: e6b08cde-61d4-4ef7-abd1-b6020fdc56c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-2d3bd-predictor-84cbc66dbf namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: d7553fae-7667-41cb-83a3-92f789216608 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'Created pod: message-dumper-raw-2d3bd-predictor-84cbc66dbf-mdbj5' metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:13Z" name: message-dumper-raw-2d3bd-predictor-84cbc66dbf.18d5e2610bc4f9c6 namespace: kserve-ci-e2e-test resourceVersion: "18467" uid: e3ef7041-66bd-465b-8702-e0deb65c9b0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: 7d4433aa-96bd-4975-9dc0-f58677f25f39 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: Scaled up replica set message-dumper-raw-2d3bd-predictor-84cbc66dbf from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:13Z" name: message-dumper-raw-2d3bd-predictor.18d5e26108a96089 namespace: kserve-ci-e2e-test resourceVersion: "18455" uid: 7bb83ac3-fe79-4c7d-aae1-6b421c628d39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:02:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: 5b2b216a-a407-416f-b1b9-7017fe31942a kind: Event lastTimestamp: "2026-09-16T19:02:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:02:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:58Z" name: message-dumper-raw-2d3bd-predictor.18d5e26486e56fde namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: a202f225-6bce-4d77-8b07-9453394c3f6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:02:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: 5b2b216a-a407-416f-b1b9-7017fe31942a kind: Event lastTimestamp: "2026-09-16T19:02:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:02:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:58Z" name: message-dumper-raw-2d3bd-predictor.18d5e264875790a5 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: 8cef7df4-56c2-4aac-8829-ac9777d811ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18645" uid: 5b2b216a-a407-416f-b1b9-7017fe31942a kind: Event lastTimestamp: "2026-09-16T19:02:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:43Z" name: message-dumper-raw-2d3bd-predictor.18d5e2680653a134 namespace: kserve-ci-e2e-test resourceVersion: "18829" uid: 85d360c6-db73-444b-a007-15c95b0fd932 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2d3bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18645" uid: 5b2b216a-a407-416f-b1b9-7017fe31942a kind: Event lastTimestamp: "2026-09-16T19:02:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:43Z" name: message-dumper-raw-2d3bd-predictor.18d5e26806b6cbd5 namespace: kserve-ci-e2e-test resourceVersion: "18832" uid: bf11c6be-2d27-434e-b6e8-2b6fe682656a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2d3bd namespace: kserve-ci-e2e-test resourceVersion: "18428" uid: bd54691f-2c0f-4127-a08a-4da2a0cb8f39 kind: Event lastTimestamp: "2026-09-16T19:02:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:02:23Z" name: message-dumper-raw-2d3bd.18d5e261086e94ae namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: f15a7351-4bb5-470d-b35c-c5de40bcba05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2d3bd namespace: kserve-ci-e2e-test resourceVersion: "18428" uid: bd54691f-2c0f-4127-a08a-4da2a0cb8f39 kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: 'Failed to update status for InferenceService "message-dumper-raw-2d3bd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-2d3bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:02:13Z" name: message-dumper-raw-2d3bd.18d5e26109c21605 namespace: kserve-ci-e2e-test resourceVersion: "18458" uid: db0614eb-f802-418c-b607-4f79ac1e13e3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2d3bd namespace: kserve-ci-e2e-test resourceVersion: "18607" uid: bd54691f-2c0f-4127-a08a-4da2a0cb8f39 kind: Event lastTimestamp: "2026-09-16T19:02:23Z" message: InferenceService [message-dumper-raw-2d3bd] is Ready metadata: creationTimestamp: "2026-09-16T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:02:23Z" name: message-dumper-raw-2d3bd.18d5e2636c0d8c5c namespace: kserve-ci-e2e-test resourceVersion: "18609" uid: 660ddb5c-4e15-4188-aada-19e9f1b385c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:11:54.275514Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23848" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cipn-predictor-9774bf55d-wlklp to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e84b6f5e36 namespace: kserve-ci-e2e-test resourceVersion: "23852" uid: d894e155-c071-4313-869f-200ae6e99592 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23857" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:54Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e86dbc04ad namespace: kserve-ci-e2e-test resourceVersion: "23866" uid: 56462b08-7adc-4019-be91-e1978b0c1bf9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:54Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e86f886309 namespace: kserve-ci-e2e-test resourceVersion: "23868" uid: 34a8fbab-326d-4e1c-a8c9-b6a2e6836112 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:55Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.102s (1.102s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-09-16T19:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:55Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e8b146e795 namespace: kserve-ci-e2e-test resourceVersion: "23877" uid: dc1548af-684e-462b-8b99-2e9b8b7faeb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:56Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e8b4507c59 namespace: kserve-ci-e2e-test resourceVersion: "23878" uid: a42be945-90fe-486c-8ef2-c37838090bc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:56Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e8b4cca1fb namespace: kserve-ci-e2e-test resourceVersion: "23879" uid: 7fb4a7f8-3ae9-41e7-ac92-430b1c230731 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:56Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e8b4d7c8d9 namespace: kserve-ci-e2e-test resourceVersion: "23880" uid: ab05977a-4016-4e3c-8896-6bc2287a4108 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:56Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e8b88287b4 namespace: kserve-ci-e2e-test resourceVersion: "23882" uid: 0652c235-37b1-4c0a-98e9-9a36fb306d14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:11:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:56Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e2e8b8ff024f namespace: kserve-ci-e2e-test resourceVersion: "23883" uid: 74f50083-3ee1-4505-b673-f0a5687c7b89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:13:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:13:49Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e303251ea438 namespace: kserve-ci-e2e-test resourceVersion: "24810" uid: 470ddbe7-4abb-4b3b-80e3-e0ec0520dea1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp namespace: kserve-ci-e2e-test resourceVersion: "23851" uid: 71bd085e-94af-46bc-9e18-c8aff7a0d879 kind: Event lastTimestamp: "2026-09-16T19:13:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:13:49Z" name: message-dumper-raw-cipn-predictor-9774bf55d-wlklp.18d5e303251f2763 namespace: kserve-ci-e2e-test resourceVersion: "24811" uid: 1f25e0ad-275d-4d8a-8ad2-0999e9f54e7c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-cipn-predictor-9774bf55d namespace: kserve-ci-e2e-test resourceVersion: "23837" uid: 653d2fa2-17c2-42ae-9c3c-791668c9f4d5 kind: Event lastTimestamp: "2026-09-16T19:11:54Z" message: 'Created pod: message-dumper-raw-cipn-predictor-9774bf55d-wlklp' metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn-predictor-9774bf55d.18d5e2e84a7f8250 namespace: kserve-ci-e2e-test resourceVersion: "23849" uid: c3af2ac6-83db-42ce-b3e6-929facf5bf0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "23835" uid: 2d0675b5-40e4-4948-8c52-78d9f76b6ab5 kind: Event lastTimestamp: "2026-09-16T19:11:54Z" message: Scaled up replica set message-dumper-raw-cipn-predictor-9774bf55d from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn-predictor.18d5e2e845f9841c namespace: kserve-ci-e2e-test resourceVersion: "23840" uid: a5d2e4c4-165c-4661-8a13-74da854f638e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "23836" uid: 8023b32a-e8db-45e5-b5a5-c2e7ee609af1 kind: Event lastTimestamp: "2026-09-16T19:12:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:09Z" name: message-dumper-raw-cipn-predictor.18d5e2ebc4f31af4 namespace: kserve-ci-e2e-test resourceVersion: "23978" uid: ff0201ad-87ba-4a21-b479-5a3370684335 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "23836" uid: 8023b32a-e8db-45e5-b5a5-c2e7ee609af1 kind: Event lastTimestamp: "2026-09-16T19:12:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:09Z" name: message-dumper-raw-cipn-predictor.18d5e2ebc560d742 namespace: kserve-ci-e2e-test resourceVersion: "23980" uid: 074c516a-2afb-4b5d-acef-b996eeaf84a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: 8023b32a-e8db-45e5-b5a5-c2e7ee609af1 kind: Event lastTimestamp: "2026-09-16T19:12:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:24Z" name: message-dumper-raw-cipn-predictor.18d5e2ef44257b33 namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 54dedfa2-3384-4de1-9abb-49cb9c129201 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: 8023b32a-e8db-45e5-b5a5-c2e7ee609af1 kind: Event lastTimestamp: "2026-09-16T19:12:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:12:24Z" name: message-dumper-raw-cipn-predictor.18d5e2ef44ad264e namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: ccee4a7d-7893-42d1-9a77-710f9ceafbb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "23828" uid: 83e5294a-7526-4d13-b240-acd5b3ee7757 kind: Event lastTimestamp: "2026-09-16T19:12:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:12:04Z" name: message-dumper-raw-cipn.18d5e2e845ee672f namespace: kserve-ci-e2e-test resourceVersion: "23948" uid: a5ef3897-315f-454d-8c5a-a6e427f26d4a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "23828" uid: 83e5294a-7526-4d13-b240-acd5b3ee7757 kind: Event lastTimestamp: "2026-09-16T19:11:54Z" message: 'Failed to update status for InferenceService "message-dumper-raw-cipn": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cipn": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn.18d5e2e846a4233e namespace: kserve-ci-e2e-test resourceVersion: "23858" uid: b640ad1c-355d-4db6-9980-232d59753e59 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "23832" uid: 83e5294a-7526-4d13-b240-acd5b3ee7757 kind: Event lastTimestamp: "2026-09-16T19:11:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cipn": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:11:54Z" name: message-dumper-raw-cipn.18d5e2e84de26338 namespace: kserve-ci-e2e-test resourceVersion: "23859" uid: 2c2a20ac-d272-480f-92fc-ceba72a10b4e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:12:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "23946" uid: 83e5294a-7526-4d13-b240-acd5b3ee7757 kind: Event lastTimestamp: "2026-09-16T19:12:04Z" message: InferenceService [message-dumper-raw-cipn] is Ready metadata: creationTimestamp: "2026-09-16T19:12:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:12:04Z" name: message-dumper-raw-cipn.18d5e2eab7988a99 namespace: kserve-ci-e2e-test resourceVersion: "23947" uid: 4b9b03ae-bf61-4232-9e42-16ead6b14d4c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:00:02.784140Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-f7202-69cb46ffc9-cdhvb to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:00:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:00:02Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e242a33e9fdd namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: f685b359-8333-4b8e-a9f3-11b04c3f50a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:03Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-f7202-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:03Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e242b1048789 namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: 6f26b86c-b872-4688-8fab-e2bb5ccbc209 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:03Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:03Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:00:03Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e242e92bfae6 namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: 882112d5-cba0-4a8b-a81b-ff542c38637f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f7202} kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:03Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:ddfcdf9928d8d607cb5523f80a587eb40baa6b36b493786c1b79430b33dd6f8d" metadata: creationTimestamp: "2026-09-16T19:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:03Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e242eaa6ff0e namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: 227af975-0894-491f-94bd-7835c2b78bc0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f7202} kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:ddfcdf9928d8d607cb5523f80a587eb40baa6b36b493786c1b79430b33dd6f8d" in 2.105s (2.105s including waiting). Image size: 211458663 bytes.' metadata: creationTimestamp: "2026-09-16T19:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:06Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e243681f85d5 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 655778b4-ef58-4cf6-a308-da7143953289 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f7202} kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:06Z" message: 'Created container: model-chainer-raw-f7202' metadata: creationTimestamp: "2026-09-16T19:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:06Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e2436b2f54ca namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 724122a1-5f51-4ee5-b26e-2a8b651d3a3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f7202} kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:06Z" message: Started container model-chainer-raw-f7202 metadata: creationTimestamp: "2026-09-16T19:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:06Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e2436baaad05 namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: ec284f50-c97a-488d-8382-88c2dac213cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f7202} kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:12Z" message: Stopping container model-chainer-raw-f7202 metadata: creationTimestamp: "2026-09-16T19:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:12Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e244fc053152 namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: 00dd9cfb-a459-4c01-92ec-734a424f27cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f7202} kind: Pod name: model-chainer-raw-f7202-69cb46ffc9-cdhvb namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 7793990c-9ca6-45d3-a8f7-dbdf4b7517a7 kind: Event lastTimestamp: "2026-09-16T19:00:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:42Z" name: model-chainer-raw-f7202-69cb46ffc9-cdhvb.18d5e24608467422 namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: 0df80392-d415-4e21-b004-c3c4a0136203 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-f7202-69cb46ffc9 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: d139fca4-3836-4b13-b77e-42a685f4cc4e kind: Event lastTimestamp: "2026-09-16T19:00:02Z" message: 'Created pod: model-chainer-raw-f7202-69cb46ffc9-cdhvb' metadata: creationTimestamp: "2026-09-16T19:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:02Z" name: model-chainer-raw-f7202-69cb46ffc9.18d5e242a2d45a1f namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 05fbdc40-0406-487f-b995-96ee6ef65038 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: b8287800-ccef-442d-ac8e-a09cd5495e6a kind: Event lastTimestamp: "2026-09-16T19:00:02Z" message: Scaled up replica set model-chainer-raw-f7202-69cb46ffc9 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:02Z" name: model-chainer-raw-f7202.18d5e242a1fa90ee namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: 9ce966b4-47c3-4f0b-8d1c-5a11445e146b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: f381cdfc-6b2e-4abb-bc0b-bd311ea7b0be kind: Event lastTimestamp: "2026-09-16T19:00:02Z" message: InferenceGraph [model-chainer-raw-f7202] is Ready metadata: creationTimestamp: "2026-09-16T19:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:02Z" name: model-chainer-raw-f7202.18d5e242a6f234e0 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: 7a7778b7-f827-4abc-87b3-de97eefd984d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: f381cdfc-6b2e-4abb-bc0b-bd311ea7b0be kind: Event lastTimestamp: "2026-09-16T19:00:02Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-f7202": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-f7202": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:02Z" name: model-chainer-raw-f7202.18d5e242a91f4553 namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: 72c98628-b833-4d52-ac6e-9a0457cd56af reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-f7202 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: f381cdfc-6b2e-4abb-bc0b-bd311ea7b0be kind: Event lastTimestamp: "2026-09-16T19:00:02Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-f7202": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:02Z" name: model-chainer-raw-f7202.18d5e242a91f8c89 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: 42e4e226-6bd9-4ef8-841a-0edb032dfd81 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:02:03.132066Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18307" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e25ea88a3be1 namespace: kserve-ci-e2e-test resourceVersion: "18312" uid: 5d1b9e52-1830-4244-988d-888cf6b0ae16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18316" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e25eca749f02 namespace: kserve-ci-e2e-test resourceVersion: "18327" uid: 442f66a9-c721-4f69-9cfd-ff137b440ece reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9ebe2} kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ddfcdf9928d8d607cb5523f80a587eb40baa6b36b493786c1b79430b33dd6f8d" already present on machine metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e25ecc23b683 namespace: kserve-ci-e2e-test resourceVersion: "18329" uid: 5e4b500e-4008-478a-8729-fb9eafa6e139 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9ebe2} kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: 'Created container: model-chainer-raw-hpa-9ebe2' metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e25ecfd4ce3a namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: 0971e79e-3d5d-40d7-acf3-f373658da6cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9ebe2} kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: Started container model-chainer-raw-hpa-9ebe2 metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e25ed051d086 namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: eb052c25-bb2f-40b3-b710-c6435a5d38ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9ebe2} kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: "2026-09-16T19:02:13Z" message: Stopping container model-chainer-raw-hpa-9ebe2 metadata: creationTimestamp: "2026-09-16T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:13Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e26100e2ac94 namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 8ddebf00-132d-49b3-906e-d8e86f266f53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9ebe2} kind: Pod name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: d07b067d-5b13-4fa1-98f7-d95aeb40f11d kind: Event lastTimestamp: "2026-09-16T19:02:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:02:39Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf.18d5e26165c4caf7 namespace: kserve-ci-e2e-test resourceVersion: "18792" uid: a4a21e63-692b-4131-b2e4-12350d2c0159 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-9ebe2-7fcdd4444c namespace: kserve-ci-e2e-test resourceVersion: "18304" uid: 1d050e0a-6350-40f0-b96c-9954c7d5379c kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: 'Created pod: model-chainer-raw-hpa-9ebe2-7fcdd4444c-bhswf' metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2-7fcdd4444c.18d5e25ea818fecf namespace: kserve-ci-e2e-test resourceVersion: "18308" uid: 049b857e-1f6b-4eca-b239-885b9fce056a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "18302" uid: cb5091ad-9f2b-4069-b0b6-a497a2b7c539 kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: Scaled up replica set model-chainer-raw-hpa-9ebe2-7fcdd4444c from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2.18d5e25ea75f35a7 namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: c02d1ecd-5342-4069-ba40-94a50a43535b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:02:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-9ebe2 namespace: kserve-ci-e2e-test resourceVersion: "18320" uid: bb6061c3-2a3a-481e-bb1b-df4e5f07df67 kind: Event lastTimestamp: "2026-09-16T19:02:03Z" message: InferenceGraph [model-chainer-raw-hpa-9ebe2] is Ready metadata: creationTimestamp: "2026-09-16T19:02:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:02:03Z" name: model-chainer-raw-hpa-9ebe2.18d5e25eacc74e15 namespace: kserve-ci-e2e-test resourceVersion: "18322" uid: ec99a7e8-c498-4138-94ae-0f6d697762ba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:04:01.936251Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-15dec-predictor-65988d47cd-bnd4v to ip-10-0-143-139.ec2.internal metadata: creationTimestamp: "2026-09-16T19:04:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:04:01Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27a51d23d3b namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: f88f7a22-502c-41fe-8afd-9e7de5a3bc01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:02Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:02Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:04:02Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27a73c0be17 namespace: kserve-ci-e2e-test resourceVersion: "19659" uid: 0e0d687a-1677-45a0-8b54-d966378f8469 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:02Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27a75ac19b9 namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: d2cf5989-9dba-4370-818a-602598466433 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:02Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27a795e9341 namespace: kserve-ci-e2e-test resourceVersion: "19664" uid: 97319772-725d-46f1-8bd9-e7f347271da0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:02Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27a79d19f9e namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: 0a24090c-8f26-4fca-8dca-8a98b00fb160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:06Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:06Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27b6431e953 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 9ba515e7-4b6a-499a-9a5e-60542a3e8929 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:06Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27b67cdac62 namespace: kserve-ci-e2e-test resourceVersion: "19707" uid: 434dc4f4-9f3f-491c-890a-dd88aaf9e384 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:06Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27b68444271 namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: 50fd98b7-3247-4d4e-81f3-26d2626a770e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:06Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27b684e1800 namespace: kserve-ci-e2e-test resourceVersion: "19709" uid: 62771a1a-5ad0-4c8c-a44e-b0f15cb0bc00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:06Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27b6b558aed namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: 03dc535e-c108-4119-b660-804ab26bdcc5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:04:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:04:06Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27b6bc614d7 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: ec61ae80-e721-4b79-94e0-93b7e0255f8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:04:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:05:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:33Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e27bdc1c2d0d namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 593bdef7-b6db-43ee-8edf-02bc51637014 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:05:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:32Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e28f4a716cf3 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 5027ba20-e3f3-466c-b07f-7ba437c0dfdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:05:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:32Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e28f4a723372 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 0f53cd14-8dcf-40c8-b132-9277686f879f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 89b3b5ad-9fa1-4615-97e2-583d2c5260a3 kind: Event lastTimestamp: "2026-09-16T19:05:34Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:34Z" name: raw-sklearn-15dec-predictor-65988d47cd-bnd4v.18d5e28fe257aa24 namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: cc94d28d-2cd0-44c5-8e2e-7f10483e8287 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-139.ec2.internal source: component: kubelet host: ip-10-0-143-139.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-15dec-predictor-65988d47cd namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: c0a17162-22d8-4a48-95cf-9601977740df kind: Event lastTimestamp: "2026-09-16T19:04:01Z" message: 'Created pod: raw-sklearn-15dec-predictor-65988d47cd-bnd4v' metadata: creationTimestamp: "2026-09-16T19:04:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:01Z" name: raw-sklearn-15dec-predictor-65988d47cd.18d5e27a51517825 namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: 5502300b-c024-46c2-ab79-516f68c73364 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-15dec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: df9611e9-fc2b-4696-ba70-a8aecc5092f6 kind: Event lastTimestamp: "2026-09-16T19:04:01Z" message: Scaled up replica set raw-sklearn-15dec-predictor-65988d47cd from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:04:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:01Z" name: raw-sklearn-15dec-predictor.18d5e27a4d69fe37 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 1297a09c-4cdc-4f49-bf48-569bd0bf5b85 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:04:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-15dec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: edceb725-b03d-4bd9-8e9a-7f131c4683bf kind: Event lastTimestamp: "2026-09-16T19:04:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:46Z" name: raw-sklearn-15dec-predictor.18d5e27dcbdff1c8 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 19d81639-002f-49da-9dd0-6978aae9c241 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:04:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-15dec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: edceb725-b03d-4bd9-8e9a-7f131c4683bf kind: Event lastTimestamp: "2026-09-16T19:04:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:04:46Z" name: raw-sklearn-15dec-predictor.18d5e27dcc558330 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 92bc03be-2204-46bf-8939-7b14af2b4314 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:05:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-15dec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: edceb725-b03d-4bd9-8e9a-7f131c4683bf kind: Event lastTimestamp: "2026-09-16T19:05:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:31Z" name: raw-sklearn-15dec-predictor.18d5e2884978cd9e namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 6afab4ec-664b-4198-ae30-303853bc43b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:05:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-15dec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: edceb725-b03d-4bd9-8e9a-7f131c4683bf kind: Event lastTimestamp: "2026-09-16T19:05:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:31Z" name: raw-sklearn-15dec-predictor.18d5e28849e3fef6 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 7a6c5fe2-de46-4ba8-a42a-44123b4f6214 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:04:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-15dec namespace: kserve-ci-e2e-test resourceVersion: "19617" uid: 23f3c069-d617-43cb-89d9-c6588f277a51 kind: Event lastTimestamp: "2026-09-16T19:05:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:04:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:05:23Z" name: raw-sklearn-15dec.18d5e27a4d69ce9c namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 3bafba7f-1647-4847-ab7f-895d37641a62 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:04:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-15dec namespace: kserve-ci-e2e-test resourceVersion: "19617" uid: 23f3c069-d617-43cb-89d9-c6588f277a51 kind: Event lastTimestamp: "2026-09-16T19:04:01Z" message: 'Failed to update status for InferenceService "raw-sklearn-15dec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-15dec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:04:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:04:01Z" name: raw-sklearn-15dec.18d5e27a4e1160c8 namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 7b9e0541-d8a5-48f7-9507-62b22e2a7300 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-15dec namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 23f3c069-d617-43cb-89d9-c6588f277a51 kind: Event lastTimestamp: "2026-09-16T19:05:23Z" message: InferenceService [raw-sklearn-15dec] is Ready metadata: creationTimestamp: "2026-09-16T19:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:05:23Z" name: raw-sklearn-15dec.18d5e28d4ae5e1fd namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: df4f9212-2df1-43cc-bfe5-d5c36ea9ce05 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:05:37.200773Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx to ip-10-0-140-230.ec2.internal metadata: creationTimestamp: "2026-09-16T19:05:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:05:37Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e29080078029 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 85fa87ff-6ad4-4ed2-a69a-ee91fb92c121 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d9bb87789-cs7bs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:37Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-fcf1d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:37Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e2908bd170b6 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 57eb7b42-148b-4b7c-9c7b-06158b043202 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:38Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:38Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:05:38Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e290c5a8d649 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 4e263d0c-7ab8-44ac-b399-f29fec692248 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fcb6571fb352fe9791aed9eb356122a360f27099bab2f66aa434c7abe7809cf4" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:38Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e290c7b06d67 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: f34be914-4539-4660-a3f9-74f74e09e26c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T19:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:38Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e290cb1d6131 namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: f2f5e4a1-a41c-40f1-a915-611e78b8b961 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T19:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:38Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e290cb91961f namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 646fc04e-8b11-4406-80ac-9104510c97ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1988" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:42Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e291a886c401 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 46d64495-4baf-40ad-8895-05dbb7897b6f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:42Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e291acde55cc namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 12751ab8-6004-4374-8054-c5d6ce3c43a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:42Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e291ad42b619 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 2abe3eba-ee37-44f8-b98c-ef4e1d3b6a78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:42Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e291ad4d171a namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: bc43d203-673a-498b-acec-d8468e816632 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:42Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e291afe706a6 namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 432e7199-e007-4e9e-8282-3655fc255e82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:05:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:42Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e291b04e7a08 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: a67e673e-9959-41e3-bad4-2f4cc6e3833b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T19:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:07:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:07Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e292204cac00 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 328bfa57-4c03-486e-b087-e51fb032f18c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:07:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:07Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e2a57843b4bc namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 31eeebfb-4dcd-4d75-8179-6121fefa69cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:07:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:07Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e2a578446a46 namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 5e8bd8e6-5b43-46c9-9d41-a929a8e3acfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 380f3970-3d65-4430-bcdb-35aba93ce880 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:10Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx.18d5e2a62677a601 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 3107f4e8-cfe3-4ccc-a66c-6c0a6b9d0395 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-230.ec2.internal source: component: kubelet host: ip-10-0-140-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-fcf1d-predictor-558b545566 namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 57a27f85-3af3-431e-8464-f47badb238ab kind: Event lastTimestamp: "2026-09-16T19:05:37Z" message: 'Created pod: raw-sklearn-runtime-fcf1d-predictor-558b545566-26mdx' metadata: creationTimestamp: "2026-09-16T19:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:37Z" name: raw-sklearn-runtime-fcf1d-predictor-558b545566.18d5e2907f8d94de namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 2928cef8-0ecd-4cd3-bcf2-93da98c7c220 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-fcf1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: e13c193a-be14-4d6b-acb3-589a532b601c kind: Event lastTimestamp: "2026-09-16T19:05:37Z" message: Scaled up replica set raw-sklearn-runtime-fcf1d-predictor-558b545566 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:37Z" name: raw-sklearn-runtime-fcf1d-predictor.18d5e2907c1b67f8 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 37353e48-dbed-4a7e-9372-cc7c2a3eae39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:05:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fcf1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 2908b19d-0db2-4859-9b8e-a75731cd4402 kind: Event lastTimestamp: "2026-09-16T19:06:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:07Z" name: raw-sklearn-runtime-fcf1d-predictor.18d5e293fa959561 namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 2fd79ec2-a5ab-4f86-9022-4c87d95c46d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:05:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fcf1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 2908b19d-0db2-4859-9b8e-a75731cd4402 kind: Event lastTimestamp: "2026-09-16T19:06:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:07Z" name: raw-sklearn-runtime-fcf1d-predictor.18d5e293faf316a4 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 6dde44ed-5e0b-4b8b-8c01-55a9f247e3eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:06:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fcf1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 2908b19d-0db2-4859-9b8e-a75731cd4402 kind: Event lastTimestamp: "2026-09-16T19:07:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:07Z" name: raw-sklearn-runtime-fcf1d-predictor.18d5e29af8d8758d namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 369eb785-1bfc-4c80-bc02-ad0d34032a98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:06:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fcf1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 2908b19d-0db2-4859-9b8e-a75731cd4402 kind: Event lastTimestamp: "2026-09-16T19:07:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:07Z" name: raw-sklearn-runtime-fcf1d-predictor.18d5e29af9438b0a namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 862ea31d-3666-4cdf-a633-35fa9da61665 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T19:05:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-fcf1d namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: 41e25c81-eafb-4db3-96ec-ab02faa058e7 kind: Event lastTimestamp: "2026-09-16T19:06:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:57Z" name: raw-sklearn-runtime-fcf1d.18d5e2907c185665 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: a5261e59-3a3d-482b-acdf-a7529508ca4e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-fcf1d namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: 41e25c81-eafb-4db3-96ec-ab02faa058e7 kind: Event lastTimestamp: "2026-09-16T19:05:37Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-fcf1d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-fcf1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:05:37Z" name: raw-sklearn-runtime-fcf1d.18d5e2907c9f544a namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 527a164c-3a82-4d10-8481-226d33d486e7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-fcf1d namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 41e25c81-eafb-4db3-96ec-ab02faa058e7 kind: Event lastTimestamp: "2026-09-16T19:06:57Z" message: InferenceService [raw-sklearn-runtime-fcf1d] is Ready metadata: creationTimestamp: "2026-09-16T19:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:57Z" name: raw-sklearn-runtime-fcf1d.18d5e2a34480e4c6 namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: e78c1d17-d881-45c6-904e-8297f1746b62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "25057"