--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:42:57.555409Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22840" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-76b74f4d6c-khx4g to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f0b00660a6 namespace: kserve-ci-e2e-test resourceVersion: "22844" uid: 44e3e5ad-f3c4-4e82-97cf-5524003eea63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22850" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f0d1f89230 namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: a614f6da-25a9-4d4b-b87c-773e917db62f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f0d36dbad5 namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c6697560-683a-4fab-8653-9c26a025b3d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f0d62b07d6 namespace: kserve-ci-e2e-test resourceVersion: "22873" uid: 0bff41e6-1d86-4b7b-adae-085816cd4e1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f0d69f8b8f namespace: kserve-ci-e2e-test resourceVersion: "22874" uid: 5c7f1314-671b-4123-8293-ccccb7841b43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f1d94495be namespace: kserve-ci-e2e-test resourceVersion: "22940" uid: 373e1050-9f4c-4afe-b099-034ec46a8d9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f1dccaa3de namespace: kserve-ci-e2e-test resourceVersion: "22943" uid: 5061902f-9d0c-4315-bf54-24ef06e5ad9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f1dd4b05c6 namespace: kserve-ci-e2e-test resourceVersion: "22945" uid: 67f85766-c7fe-4bf5-aadf-bd298efb8909 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f1dd58df47 namespace: kserve-ci-e2e-test resourceVersion: "22947" uid: 685b4d95-0639-459a-bca7-ed573028ad5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f1e3ee56f1 namespace: kserve-ci-e2e-test resourceVersion: "22951" uid: 02fba094-2426-49aa-8140-69d9c97dd077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f1e4b14a6e namespace: kserve-ci-e2e-test resourceVersion: "22954" uid: c3f849ec-6923-4b58-a2de-bd701334ff9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:44:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:38Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d677f21554e1f0 namespace: kserve-ci-e2e-test resourceVersion: "23722" uid: 92d57b35-1662-4076-afb0-c345101facd4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:44:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:37Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d67807f8eca2b2 namespace: kserve-ci-e2e-test resourceVersion: "23712" uid: ef033e7f-27ea-4699-94af-b49975a92eaf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:44:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:37Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d67807f8ef8cc9 namespace: kserve-ci-e2e-test resourceVersion: "23713" uid: 82bf01ab-cd2a-49bf-945e-7d27d332d1b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-khx4g namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: a71e99af-6719-4969-a1f2-e6aebf423b15 kind: Event lastTimestamp: "2026-09-18T16:44:40Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:40Z" name: isvc-canary-create-predictor-76b74f4d6c-khx4g.18d67808abae371b namespace: kserve-ci-e2e-test resourceVersion: "23746" uid: dc4ac0cc-4ee6-4845-a868-2412857464b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:42:57.496670Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22823" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-76b74f4d6c-xs4ct to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f0ac861d60 namespace: kserve-ci-e2e-test resourceVersion: "22833" uid: a872549b-65d5-453e-90d8-16756429adbf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22837" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f0ce7c3c8c namespace: kserve-ci-e2e-test resourceVersion: "22862" uid: c2628acf-3d7d-4b88-bc96-bf5d96797c09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f0d01075b3 namespace: kserve-ci-e2e-test resourceVersion: "22864" uid: 7daf4e86-520f-4646-9bc2-6d5cd06c173a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f0d2a33dff namespace: kserve-ci-e2e-test resourceVersion: "22867" uid: c925a7d5-cdcd-4494-ba33-d2e545b1514a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f0d312fcc2 namespace: kserve-ci-e2e-test resourceVersion: "22869" uid: 576bbd70-c17b-4d46-93e8-9b9090a4d485 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f1eff605fb namespace: kserve-ci-e2e-test resourceVersion: "22956" uid: 4862d969-e8ec-45ed-925a-b363ca0e5b5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f1f353ed0e namespace: kserve-ci-e2e-test resourceVersion: "22959" uid: 15c596a1-23f4-48c0-b993-0f239c5820c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f1f3c9fcd3 namespace: kserve-ci-e2e-test resourceVersion: "22960" uid: 8093fd5d-1834-47aa-981d-01e9fd0b15c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f1f3d42dfb namespace: kserve-ci-e2e-test resourceVersion: "22961" uid: 1ef647c7-43fb-4733-b392-9ce69d6549f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f1f70821cf namespace: kserve-ci-e2e-test resourceVersion: "22962" uid: 24b3b78b-abc2-47df-b7a9-13c03448a17f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f1f77e9bfd namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 11375d73-44d9-4c1a-9a45-610eba88305e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T16:43:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:44:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:10Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d677f2682a6b6f namespace: kserve-ci-e2e-test resourceVersion: "23463" uid: 3aff3b95-3859-4f21-a272-9ad7d8b805f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d67803f70c6939 namespace: kserve-ci-e2e-test resourceVersion: "23547" uid: a93923b9-1ed4-4cb1-aae6-0d65e125eaa6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d67803f70d309d namespace: kserve-ci-e2e-test resourceVersion: "23548" uid: e99b1d4c-1086-4d57-861c-f60520feb6b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-predictor-76b74f4d6c-xs4ct namespace: kserve-ci-e2e-test resourceVersion: "22828" uid: e42b2760-c8a8-43f2-abc6-413d29f7c34c kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create-predictor-76b74f4d6c-xs4ct.18d678041a539b94 namespace: kserve-ci-e2e-test resourceVersion: "23570" uid: 43ce3dd8-1a18-4b7e-ade6-9ce8ebd55f6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-predictor-76b74f4d6c namespace: kserve-ci-e2e-test resourceVersion: "22812" uid: 068daf41-c1ae-41e1-bf89-33d0adedc48e kind: Event lastTimestamp: "2026-09-18T16:42:57Z" message: 'Created pod: isvc-canary-create-predictor-76b74f4d6c-xs4ct' metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-predictor-76b74f4d6c.18d677f0ac20abd9 namespace: kserve-ci-e2e-test resourceVersion: "22827" uid: bc093729-1a28-459e-8b74-465416c3b3b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-predictor-76b74f4d6c namespace: kserve-ci-e2e-test resourceVersion: "22812" uid: 068daf41-c1ae-41e1-bf89-33d0adedc48e kind: Event lastTimestamp: "2026-09-18T16:42:57Z" message: 'Created pod: isvc-canary-create-predictor-76b74f4d6c-khx4g' metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-predictor-76b74f4d6c.18d677f0af8bc3ad namespace: kserve-ci-e2e-test resourceVersion: "22841" uid: ab769944-dc74-4603-b358-dcb203c34abb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-predictor-76b74f4d6c namespace: kserve-ci-e2e-test resourceVersion: "23539" uid: 068daf41-c1ae-41e1-bf89-33d0adedc48e kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: 'Deleted pod: isvc-canary-create-predictor-76b74f4d6c-xs4ct' metadata: creationTimestamp: "2026-09-18T16:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create-predictor-76b74f4d6c.18d67803f6df11e6 namespace: kserve-ci-e2e-test resourceVersion: "23542" uid: 9cdafed6-7905-46c2-b4c5-aa7eef2fecef reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-create-predictor namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: c0471b86-9f49-4aac-8736-944dd663381e kind: Event lastTimestamp: "2026-09-18T16:42:57Z" message: Scaled up replica set isvc-canary-create-predictor-76b74f4d6c from 0 to 2 metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-predictor.18d677f0a8671bb9 namespace: kserve-ci-e2e-test resourceVersion: "22814" uid: 7b839c91-0393-444c-973a-0ff140823623 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-create-predictor namespace: kserve-ci-e2e-test resourceVersion: "23535" uid: c0471b86-9f49-4aac-8736-944dd663381e kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: Scaled down replica set isvc-canary-create-predictor-76b74f4d6c from 2 to 1 metadata: creationTimestamp: "2026-09-18T16:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create-predictor.18d67803f6724923 namespace: kserve-ci-e2e-test resourceVersion: "23540" uid: a05b8e6a-990c-410e-865a-caf2ff7bfbd6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:42:57.485996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22819" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-667f95d746-82srz to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f0abe3450c namespace: kserve-ci-e2e-test resourceVersion: "22825" uid: fe69bea9-4337-4e1f-bc1a-0bba8af66bdf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:42:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-create-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f0baa6bc54 namespace: kserve-ci-e2e-test resourceVersion: "22854" uid: 2f16b3e7-1c91-4365-a615-4a533d15e5eb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22836" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f0f2073119 namespace: kserve-ci-e2e-test resourceVersion: "22882" uid: d1187ec3-a3b7-4a1b-9eac-0778af494a75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f0f38dd9d8 namespace: kserve-ci-e2e-test resourceVersion: "22884" uid: bbec8e30-4a5c-4120-b423-2d7ef0321940 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f0f6c2cc0e namespace: kserve-ci-e2e-test resourceVersion: "22885" uid: 6f71ec28-7c9e-4327-a199-dc4b3e010507 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:58Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f0f743f929 namespace: kserve-ci-e2e-test resourceVersion: "22886" uid: 991c1f1d-ccae-465c-81f1-91ab135d2be1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f2155d4a06 namespace: kserve-ci-e2e-test resourceVersion: "22969" uid: 4381f049-38c5-472a-94c3-9f47b4e88733 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f218cb77f6 namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: 40ed7a2d-5efe-4c59-80d9-e59b41946c95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f21945eab8 namespace: kserve-ci-e2e-test resourceVersion: "22973" uid: 6930ebc6-1e5e-4301-9128-4646c744f3da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f219509a0a namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: bdb94d8a-9124-4ed2-a684-5f60866bf23e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f21c48aa88 namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 8b21dfca-2c6e-4181-8b32-978acbbc13f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:43:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:03Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f21cbf1a25 namespace: kserve-ci-e2e-test resourceVersion: "22977" uid: b112e47a-75d3-45cc-ab77-195189452484 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:43:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:44:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:40Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d677f251729229 namespace: kserve-ci-e2e-test resourceVersion: "23741" uid: bd7479c0-07a1-4899-a2e1-89e5da2bf52d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:44:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:37Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d67807f924bc3f namespace: kserve-ci-e2e-test resourceVersion: "23714" uid: 7812332f-4587-4cd4-b688-619e37eb3f67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:44:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:37Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d67807f9252a47 namespace: kserve-ci-e2e-test resourceVersion: "23716" uid: 9618f877-a8ee-48ad-a8a0-176368f7da90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-create-v2-predictor-667f95d746-82srz namespace: kserve-ci-e2e-test resourceVersion: "22820" uid: d976b2ee-0435-47ce-a33c-1995934f216d kind: Event lastTimestamp: "2026-09-18T16:44:40Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:40Z" name: isvc-canary-create-v2-predictor-667f95d746-82srz.18d67808abaf1bea namespace: kserve-ci-e2e-test resourceVersion: "23747" uid: 37d13919-32dc-4a27-8fd7-14b4faf68f0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-create-v2-predictor-667f95d746 namespace: kserve-ci-e2e-test resourceVersion: "22803" uid: c6312c51-4106-40dd-9660-74dde6b6aa5b kind: Event lastTimestamp: "2026-09-18T16:42:57Z" message: 'Created pod: isvc-canary-create-v2-predictor-667f95d746-82srz' metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-v2-predictor-667f95d746.18d677f0ab842da2 namespace: kserve-ci-e2e-test resourceVersion: "22821" uid: c8fa50a4-6488-4c22-9749-ef8c384b245e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-create-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "22802" uid: 24b03b8b-fcaf-4ec0-ae93-80e0b40b0bf9 kind: Event lastTimestamp: "2026-09-18T16:42:57Z" message: Scaled up replica set isvc-canary-create-v2-predictor-667f95d746 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:57Z" name: isvc-canary-create-v2-predictor.18d677f0a626032f namespace: kserve-ci-e2e-test resourceVersion: "22804" uid: cd75a536-c529-4e83-95db-caacb6836135 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "22794" uid: 54b6175a-565c-4d24-8594-6753554d0dc9 kind: Event lastTimestamp: "2026-09-18T16:44:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:44:28Z" name: isvc-canary-create.18d677f0a801f934 namespace: kserve-ci-e2e-test resourceVersion: "23624" uid: 1a233bca-492a-4196-a9de-0e20c3776500 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "22794" uid: 54b6175a-565c-4d24-8594-6753554d0dc9 kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: 'Failed to update status for InferenceService "isvc-canary-create": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create.18d677f0a8725189 namespace: kserve-ci-e2e-test resourceVersion: "23555" uid: 20772677-dfbd-4ad9-8b22-65812f3710aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:42:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "22794" uid: 54b6175a-565c-4d24-8594-6753554d0dc9 kind: Event lastTimestamp: "2026-09-18T16:44:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:44:20Z" name: isvc-canary-create.18d677f0a8728eed namespace: kserve-ci-e2e-test resourceVersion: "23556" uid: c30fb5c9-b59b-444c-8a97-d1f108258c6b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-create namespace: kserve-ci-e2e-test resourceVersion: "23625" uid: 54b6175a-565c-4d24-8594-6753554d0dc9 kind: Event lastTimestamp: "2026-09-18T16:44:28Z" message: InferenceService [isvc-canary-create] is Ready metadata: creationTimestamp: "2026-09-18T16:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:44:28Z" name: isvc-canary-create.18d67805d1968398 namespace: kserve-ci-e2e-test resourceVersion: "23626" uid: 2c45a0be-30fa-4cd9-8ba9-6ffad6b376bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:44:42.738691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23796" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-6546f8697f-5kh84 to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d678092d6fe8db namespace: kserve-ci-e2e-test resourceVersion: "23803" uid: 0f16f9a4-e1b8-4756-87fc-a1598f7678ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23809" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d678094f75e23e namespace: kserve-ci-e2e-test resourceVersion: "23830" uid: 557551aa-41c0-4ffb-a030-aeb25df933e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780950c5d88d namespace: kserve-ci-e2e-test resourceVersion: "23834" uid: 3e1700c8-e86e-479f-983a-3991bc956075 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d67809543d31d8 namespace: kserve-ci-e2e-test resourceVersion: "23841" uid: abfd7bc9-87a1-43a6-984d-7f692c94291b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780954b84aa9 namespace: kserve-ci-e2e-test resourceVersion: "23843" uid: b94005e4-9b37-41c0-9769-fbbf637f455a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:48Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:48Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780a67c91d37 namespace: kserve-ci-e2e-test resourceVersion: "23907" uid: 3029873e-02c0-4e9e-881a-eee36dcf3c51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:48Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780a6afc0e55 namespace: kserve-ci-e2e-test resourceVersion: "23909" uid: 74e88900-5df4-4f09-b24c-9d69d66ca843 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:48Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780a6b71e20f namespace: kserve-ci-e2e-test resourceVersion: "23910" uid: 689e4726-22a7-434c-ae51-e67a6bec3988 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:48Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780a6b7ba96d namespace: kserve-ci-e2e-test resourceVersion: "23911" uid: 593e2d11-824d-478a-90ca-00e9757953b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:48Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780a6e4269ac namespace: kserve-ci-e2e-test resourceVersion: "23914" uid: 35ba58c9-660d-41d4-b009-f280d98c2f68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:44:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:48Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780a6eb996d1 namespace: kserve-ci-e2e-test resourceVersion: "23915" uid: 3dcd30a9-ecfb-41d5-864c-88df0f85458e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:45:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:45:59Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6780aa3d8f5fb namespace: kserve-ci-e2e-test resourceVersion: "24428" uid: 494f4518-de6d-42c5-95e4-dfae033d18bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:46:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:05Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6781c72edab3e namespace: kserve-ci-e2e-test resourceVersion: "24627" uid: e7e8da45-d9b5-4e8e-a0ee-3cb1e265484f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:46:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:05Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6781c72ee1aa3 namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: f4f069c9-ee0a-4157-b102-d39cdcfddded reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-5kh84 namespace: kserve-ci-e2e-test resourceVersion: "23799" uid: f2980fd2-8c99-41c1-8612-a0bef0bb2841 kind: Event lastTimestamp: "2026-09-18T16:46:10Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:10Z" name: isvc-canary-promote-predictor-6546f8697f-5kh84.18d6781d806ed920 namespace: kserve-ci-e2e-test resourceVersion: "24771" uid: be605194-9bba-44ab-8846-79d291893033 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:44:42.798218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23810" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-6546f8697f-ddmfm to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780930fc3a95 namespace: kserve-ci-e2e-test resourceVersion: "23814" uid: 49a4e5ea-77e2-42a2-a232-25d1738148a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23821" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780952ed4c6b namespace: kserve-ci-e2e-test resourceVersion: "23836" uid: 031f93e9-b64c-4161-b662-9ac594a1a50c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780954597559 namespace: kserve-ci-e2e-test resourceVersion: "23842" uid: d4dec472-0e4c-4825-8cf1-757a7813d47b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780956fbafd9 namespace: kserve-ci-e2e-test resourceVersion: "23844" uid: 871bb161-8195-42e4-87dc-06f494c190d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d678095770fe89 namespace: kserve-ci-e2e-test resourceVersion: "23845" uid: 9c7f281b-52ba-48a4-a3b6-b70d54369f0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780a40b28cda namespace: kserve-ci-e2e-test resourceVersion: "23889" uid: a76f6ed9-de76-47fc-90fc-ac8ba3ba584e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780a4692da84 namespace: kserve-ci-e2e-test resourceVersion: "23895" uid: b8d6d8aa-4467-4533-9374-ffe703ae71f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780a471f0be0 namespace: kserve-ci-e2e-test resourceVersion: "23896" uid: d07135bf-d25b-41dd-a99b-28806d79df67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780a472a401f namespace: kserve-ci-e2e-test resourceVersion: "23897" uid: 18ea0076-5dcf-40fb-a50a-2eac56be1beb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780a4addeb23 namespace: kserve-ci-e2e-test resourceVersion: "23901" uid: a6f5d996-f6d9-4bb5-83e4-18b77ddd715a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780a4b4e6736 namespace: kserve-ci-e2e-test resourceVersion: "23903" uid: c7dbe59c-f943-4b68-8b10-145cde494811 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:46:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:15Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6780ab8a0f62f namespace: kserve-ci-e2e-test resourceVersion: "24897" uid: 53b232ad-f20f-46ce-9963-23b37000881e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:46:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6781e281914f9 namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: c5ec8690-7858-4bfd-963d-ff9d32eb15dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:46:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6781e28198dba namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: 8b03d7a7-6317-4ec4-b895-22a87260a516 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-predictor-6546f8697f-ddmfm namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: dcf475c8-5832-4bd2-b57e-5e04415df9ba kind: Event lastTimestamp: "2026-09-18T16:46:15Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:15Z" name: isvc-canary-promote-predictor-6546f8697f-ddmfm.18d6781ebedc53c3 namespace: kserve-ci-e2e-test resourceVersion: "24896" uid: e9fa1267-f3d0-46b8-af82-accc4ea8ef6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-predictor-6546f8697f namespace: kserve-ci-e2e-test resourceVersion: "23784" uid: 254b3a7e-5176-48bf-bf77-084a19061a18 kind: Event lastTimestamp: "2026-09-18T16:44:42Z" message: 'Created pod: isvc-canary-promote-predictor-6546f8697f-5kh84' metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-predictor-6546f8697f.18d678092d0739c2 namespace: kserve-ci-e2e-test resourceVersion: "23798" uid: 5cf93c2b-058c-447a-90bc-115830d9eb66 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-predictor-6546f8697f namespace: kserve-ci-e2e-test resourceVersion: "23784" uid: 254b3a7e-5176-48bf-bf77-084a19061a18 kind: Event lastTimestamp: "2026-09-18T16:44:42Z" message: 'Created pod: isvc-canary-promote-predictor-6546f8697f-ddmfm' metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-predictor-6546f8697f.18d67809305ae9ad namespace: kserve-ci-e2e-test resourceVersion: "23811" uid: d55ea487-6de9-4d4a-b46b-c5f0615df3dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-predictor-6546f8697f namespace: kserve-ci-e2e-test resourceVersion: "24619" uid: 254b3a7e-5176-48bf-bf77-084a19061a18 kind: Event lastTimestamp: "2026-09-18T16:46:05Z" message: 'Deleted pod: isvc-canary-promote-predictor-6546f8697f-5kh84' metadata: creationTimestamp: "2026-09-18T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:46:05Z" name: isvc-canary-promote-predictor-6546f8697f.18d6781c72c7e4eb namespace: kserve-ci-e2e-test resourceVersion: "24623" uid: a6e59a48-e9a8-4565-80f8-71d6c6cfd481 reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-predictor namespace: kserve-ci-e2e-test resourceVersion: "23783" uid: d20bb585-d026-47ac-b91f-164818516ee9 kind: Event lastTimestamp: "2026-09-18T16:44:42Z" message: Scaled up replica set isvc-canary-promote-predictor-6546f8697f from 0 to 2 metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-predictor.18d6780929a231e8 namespace: kserve-ci-e2e-test resourceVersion: "23786" uid: 32292298-09d3-4679-b67d-5514a80939d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-predictor namespace: kserve-ci-e2e-test resourceVersion: "24615" uid: d20bb585-d026-47ac-b91f-164818516ee9 kind: Event lastTimestamp: "2026-09-18T16:46:05Z" message: Scaled down replica set isvc-canary-promote-predictor-6546f8697f from 2 to 1 metadata: creationTimestamp: "2026-09-18T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:46:05Z" name: isvc-canary-promote-predictor.18d6781c724e7d6f namespace: kserve-ci-e2e-test resourceVersion: "24620" uid: 2fe5ef51-ca78-42e4-941d-d68f252e21fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:46:12.911928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24852" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:46:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781e2c2fb022 namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 90ebac29-0e8a-431c-9620-5aaf675261a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24860" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:13Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:46:13Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781e4e450ce7 namespace: kserve-ci-e2e-test resourceVersion: "24866" uid: 42ad563a-dbc2-4af2-a87d-49c26a73c19b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:13Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781e50eb9af5 namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 2ce31f36-4a1a-4a9a-a966-8be567cae285 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:13Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781e53c42748 namespace: kserve-ci-e2e-test resourceVersion: "24869" uid: 96549686-f9cd-4f5c-b6ba-1a68deafcdc8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:13Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781e543a45a9 namespace: kserve-ci-e2e-test resourceVersion: "24870" uid: 82721f4a-66dd-464d-a11a-2525153e8f6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:17Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:17Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781f3762149e namespace: kserve-ci-e2e-test resourceVersion: "24911" uid: fa629825-3fa8-41a8-a86f-074952a14ff1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:17Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781f3a966ac0 namespace: kserve-ci-e2e-test resourceVersion: "24914" uid: f5244b4b-9828-4a08-adf0-609abe85c850 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:17Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781f3b0d49d0 namespace: kserve-ci-e2e-test resourceVersion: "24915" uid: e4835c4d-3cbe-48f9-9ddf-a3ae385d4037 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:17Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781f3b185f33 namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: 7967da1a-97c8-4230-af1f-c72bfae048b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:17Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781f3e50d070 namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: edcc40bb-372f-4981-8982-155d2dfb8109 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:46:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:46:17Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781f3ec7ec31 namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 1baf8b67-a5cb-4f49-8ec6-ba5c63a6fbb0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:47:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:45Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d6781faf3ad724 namespace: kserve-ci-e2e-test resourceVersion: "25569" uid: e60b5d70-2f6c-4b3a-83c2-6282d30d5237 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:47:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:47:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d67833297f3e05 namespace: kserve-ci-e2e-test resourceVersion: "25543" uid: dbf05fb2-e9a1-41a4-8687-d0c1cc694884 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:47:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:47:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d67833297fd20e namespace: kserve-ci-e2e-test resourceVersion: "25545" uid: 7a62fd85-ce66-4c5b-85b9-dfae91a1e61d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 06458284-0347-4d9c-84bb-8a0e529f9d1b kind: Event lastTimestamp: "2026-09-18T16:47:45Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:45Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt.18d67833b57cf7d5 namespace: kserve-ci-e2e-test resourceVersion: "25568" uid: dc900630-3187-4da2-a7bf-d9517318886a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:44:42.729203Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23792" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d678092cdf1c65 namespace: kserve-ci-e2e-test resourceVersion: "23800" uid: 3f4dcf02-72dc-412f-ab41-3728ef882da8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23808" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d678094ef17d19 namespace: kserve-ci-e2e-test resourceVersion: "23829" uid: e6d3309d-3f79-41c4-9f42-5cbca96fdbb9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d678095059b1b9 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: a02212ae-ae1a-4f7a-9aec-a284896e9c24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d67809532cb7e5 namespace: kserve-ci-e2e-test resourceVersion: "23837" uid: db5de7f8-e765-4beb-ace4-29d8874bb582 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780953a2d01e namespace: kserve-ci-e2e-test resourceVersion: "23839" uid: c9742d00-2383-4bdf-97e6-41d88c1d0a7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780a409ad247 namespace: kserve-ci-e2e-test resourceVersion: "23888" uid: c9d18b21-760f-470c-a72e-ae1bc51535e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780a46887449 namespace: kserve-ci-e2e-test resourceVersion: "23893" uid: 47af2fa4-98ed-4535-b2a5-85e40e67cf64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780a472c8992 namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1db0c579-69e5-4c01-b96b-d2fff25c9061 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780a4735bbd5 namespace: kserve-ci-e2e-test resourceVersion: "23899" uid: b1af47e1-3a46-47c2-823a-5b86e3a235ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780a4af529a5 namespace: kserve-ci-e2e-test resourceVersion: "23902" uid: c94055f4-ab4e-4d69-8b68-f4984b018df9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:44:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:44:47Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780a4b673fe6 namespace: kserve-ci-e2e-test resourceVersion: "23904" uid: 89b618ad-8396-4da9-a890-0e300be26e2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:47:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:45Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d6780ab8a1712d namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: 4722dd15-6a8d-47e1-b4bb-2bfb8be14b72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:47:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:47:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d67833293829d4 namespace: kserve-ci-e2e-test resourceVersion: "25542" uid: 1d3bc89f-5daa-4167-8f1d-2d3b7e7a0e7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:47:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:47:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:43Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d678332938a990 namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: c566b998-5b9e-4f98-8d0f-42a3ce33af31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4 namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: b927b2ba-a937-4713-b063-a04091ea1c91 kind: Event lastTimestamp: "2026-09-18T16:47:45Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:45Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4.18d67833b344a6ef namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: e971b7ed-3866-4ffe-99c6-612a565a9c12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-v2-predictor-84fcdd9b5b namespace: kserve-ci-e2e-test resourceVersion: "23774" uid: b6bbaf55-ee09-4448-85e1-ff52f05dafff kind: Event lastTimestamp: "2026-09-18T16:44:42Z" message: 'Created pod: isvc-canary-promote-v2-predictor-84fcdd9b5b-ksdk4' metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b.18d678092c951e0e namespace: kserve-ci-e2e-test resourceVersion: "23795" uid: c56a49d6-31dc-4414-87cf-c4d3b8f55be1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-promote-v2-predictor-84fcdd9b5b namespace: kserve-ci-e2e-test resourceVersion: "24835" uid: b6bbaf55-ee09-4448-85e1-ff52f05dafff kind: Event lastTimestamp: "2026-09-18T16:46:12Z" message: 'Created pod: isvc-canary-promote-v2-predictor-84fcdd9b5b-fkvgt' metadata: creationTimestamp: "2026-09-18T16:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote-v2-predictor-84fcdd9b5b.18d6781e2b9e9c8e namespace: kserve-ci-e2e-test resourceVersion: "24853" uid: 4a20d257-3768-4a34-90e1-d55cd09046f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23773" uid: dbcd0c0b-1255-4b20-9ee3-8cdaba7dd3ab kind: Event lastTimestamp: "2026-09-18T16:44:42Z" message: Scaled up replica set isvc-canary-promote-v2-predictor-84fcdd9b5b from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:44:42Z" name: isvc-canary-promote-v2-predictor.18d6780927a2b587 namespace: kserve-ci-e2e-test resourceVersion: "23775" uid: 06ce5f13-8d7c-4f89-85b3-a8b69506e62b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-promote-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "24829" uid: dbcd0c0b-1255-4b20-9ee3-8cdaba7dd3ab kind: Event lastTimestamp: "2026-09-18T16:46:12Z" message: Scaled up replica set isvc-canary-promote-v2-predictor-84fcdd9b5b from 1 to 2 metadata: creationTimestamp: "2026-09-18T16:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote-v2-predictor.18d6781e266fe7f6 namespace: kserve-ci-e2e-test resourceVersion: "24837" uid: 632a7504-e49a-4ce2-94a9-cccfcbded403 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 20 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "23765" uid: 4b62be2a-fe23-4fba-86d7-ee75e009f279 kind: Event lastTimestamp: "2026-09-18T16:46:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote.18d67809294a5cde namespace: kserve-ci-e2e-test resourceVersion: "24861" uid: 062e0f5c-fa0d-47de-b066-83360d42f942 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "23765" uid: 4b62be2a-fe23-4fba-86d7-ee75e009f279 kind: Event lastTimestamp: "2026-09-18T16:46:05Z" message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:46:05Z" name: isvc-canary-promote.18d6780929afafd1 namespace: kserve-ci-e2e-test resourceVersion: "24635" uid: 03ed1fd5-fc62-4222-b79b-d02a2489f58b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "23765" uid: 4b62be2a-fe23-4fba-86d7-ee75e009f279 kind: Event lastTimestamp: "2026-09-18T16:46:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:46:05Z" name: isvc-canary-promote.18d6780929b00bf5 namespace: kserve-ci-e2e-test resourceVersion: "24636" uid: 63c874a3-27a1-4e41-a745-0e92e1cb5130 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: 4b62be2a-fe23-4fba-86d7-ee75e009f279 kind: Event lastTimestamp: "2026-09-18T16:47:35Z" message: InferenceService [isvc-canary-promote] is Ready metadata: creationTimestamp: "2026-09-18T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:47:35Z" name: isvc-canary-promote.18d6781c799bf1be namespace: kserve-ci-e2e-test resourceVersion: "25473" uid: 1b7db6a4-533f-4836-8534-03cf475ce0f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:46:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-promote namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: 4b62be2a-fe23-4fba-86d7-ee75e009f279 kind: Event lastTimestamp: "2026-09-18T16:46:12Z" message: 'InferenceService [isvc-canary-promote] is no longer Ready because of: PredictorReady, Ready, Stopped' metadata: creationTimestamp: "2026-09-18T16:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:46:12Z" name: isvc-canary-promote.18d6781e2b02d953 namespace: kserve-ci-e2e-test resourceVersion: "24851" uid: d76152e1-5b63-49a0-8b87-51c82b95c015 reason: InferenceServiceNotReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:32:06.861736Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-7f7b644c9-rdm8j to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d677592fa8ef44 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: d6d5bc68-c4e3-4611-8b88-32ee45b2ae5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775951e6d68a namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 3db4db3e-7799-4937-8919-f0982a9affd6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d67759541ca754 namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: 8ec784ca-bf14-4100-9912-b8a2c68d2827 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" in 4.264s (4.264s including waiting). Image size: 308174268 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775a5250518e namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: f25f791d-3849-48b7-b510-5c0b9fbbb1be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775a56335b2e namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 2211f0dc-1a79-41f4-ad81-7a0bb75f17f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775a56fb1862 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 0a9805ee-869a-4f49-9c14-5827d12eb5ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:15Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:15Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775b3a077d36 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: d3563b3a-875a-407b-8bf7-e96ab8afb3a7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" in 16.787s (16.787s including waiting). Image size: 1607081368 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775f229ef0e3 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: a0540613-d1f2-43fb-ad90-8af976bbfebf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775f2740fcad namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: df2be640-e627-4678-b9e6-6b86e83a266c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775f284f0346 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 16a5e91a-bf3d-4389-9d94-b2c4f8cff2a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775f285ca410 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 44e75f70-4acb-49e9-94f0-471293b98cfd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.61s (2.61s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775fc3fca26c namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: 3e9c9ecb-5dce-46ef-a34e-877f09cebcd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775fc836dc38 namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 95fc3477-c15b-4bac-b487-4a34852fec01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775fc8b219b2 namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: f6b8536d-29e8-4917-b2a5-5fb7690260b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:33:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:48Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d6775fe9d09e04 namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: 27e4b65d-f493-4dd7-893e-e4c01ee51b96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:33:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:48Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d67770d9125ea4 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: 407fe964-036a-4417-8058-a9f0c57967f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:33:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:48Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d67770d9133a5d namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: 0a0afd00-c3ea-4769-8800-f0ccbd4d194a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 4a9c7bc7-e80b-4d5a-90c8-47c0f5261103 kind: Event lastTimestamp: "2026-09-18T16:33:51Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:51Z" name: isvc-canary-rollback-predictor-7f7b644c9-rdm8j.18d677719c26e524 namespace: kserve-ci-e2e-test resourceVersion: "17232" uid: bee999c1-5a0f-4ef7-b503-636ce14e0968 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:32:06.772112Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-7f7b644c9-sxhhv to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d677592a514f78 namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: eeebf2f9-65bc-4f74-affe-86fc76cbf5df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d677594c7ad074 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 2da82e50-ed78-42d0-9e8b-b4b6b6cc2720 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d677594fc64518 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: cfb79618-374c-44b9-be90-192b24f19f17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d67759535dcaf9 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 2c95b344-0bc6-4398-ba2d-53d963c74a41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775953cdd142 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 98c104da-5cbc-428b-89f7-cfd773cfa0d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775a32803e70 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: a4a34e77-02cc-4b9b-aea7-c4a926ebcd54 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:28Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" in 17.502s (17.502s including waiting). Image size: 1607081368 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:28Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775e45bcebc2 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 4b72124a-9c17-4ab3-8cd9-6672f9c88ba5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:28Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775e4c564708 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: b69fd6f5-9e92-4f90-a28b-2812b33851f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:28Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775e4d673471 namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: c83d205c-1640-482a-9fd7-d423318cb5cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:28Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775e4d744ea9 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 1b3ae09a-04b8-4b09-b099-e7bc3ba6ba31 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.594s (2.594s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:31Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775ee81a1f0d namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 1d64076f-6b7d-49a0-ab18-7e361052297d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:31Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775eeaf8a29f namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: f35af6b6-ee5a-4fe3-b203-5b66355bee51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:32:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:31Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775eeb5bf638 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: d5b2c501-f79e-420d-8916-94002240b88b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:35:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:27Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d6775f1eca28dd namespace: kserve-ci-e2e-test resourceVersion: "18182" uid: a64e2e0a-6038-4544-9c21-56e60ad7bf03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:35:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:26Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d67787ba1385f0 namespace: kserve-ci-e2e-test resourceVersion: "18173" uid: d2291fe9-130a-4c20-8b31-c13b748f43ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:35:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:26Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d67787ba14a1b1 namespace: kserve-ci-e2e-test resourceVersion: "18175" uid: bcfaa2fc-5d19-46da-94df-e560c18dc2ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 89932c02-ee32-4fd0-8e70-631066926d34 kind: Event lastTimestamp: "2026-09-18T16:35:28Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:35:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:28Z" name: isvc-canary-rollback-predictor-7f7b644c9-sxhhv.18d67788196e4771 namespace: kserve-ci-e2e-test resourceVersion: "18188" uid: 38d09c02-6507-4ed1-8ced-8e819330de15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:33:56.784544Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-7f7b644c9-wsgs4 to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:33:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67772c791d014 namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 9388b586-cced-49e6-9551-a340a3059177 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:33:57Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:33:57Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67772e97b54e6 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: b4707ebc-56de-4eb4-a999-e5558a6e7aad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:33:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:57Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67772eadc29bc namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: d5f7b58d-765b-4ac0-baf8-9754dcf0857c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:33:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:57Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67772ed58f936 namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: 902658d7-b73b-41c9-9320-618354bc4783 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:33:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:57Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67772ede1cddf namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: 0846a868-a129-4151-b1fd-59e639c766de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67774065f8ad2 namespace: kserve-ci-e2e-test resourceVersion: "17407" uid: 39b9a359-496c-41ba-b783-c55aac71b25d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d677740931650d namespace: kserve-ci-e2e-test resourceVersion: "17416" uid: b726d1e6-6c81-49e4-b562-23caac6cceca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d6777409ab09a4 namespace: kserve-ci-e2e-test resourceVersion: "17419" uid: e6845e04-bf82-4bc9-9dfc-e8d0d870e094 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d6777409b9f618 namespace: kserve-ci-e2e-test resourceVersion: "17422" uid: 5976f841-cbf7-4813-aea2-5c6934544f4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d677740cb48e69 namespace: kserve-ci-e2e-test resourceVersion: "17429" uid: c8d13c3b-918c-4870-a154-05e8e166cfce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d677740d412560 namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 6f5ecf00-7bcc-4533-ab16-83909af300ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:34:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:35:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:30Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d677747e6073d3 namespace: kserve-ci-e2e-test resourceVersion: "18204" uid: 0669f00c-79d9-433d-9db9-8ffcd45ddfb3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:35:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:26Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67787b9dca75d namespace: kserve-ci-e2e-test resourceVersion: "18172" uid: 862dd733-aeb9-4c65-a6af-babd9bbff950 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:35:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:26Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d67787b9dd30f3 namespace: kserve-ci-e2e-test resourceVersion: "18174" uid: 6a09b946-f730-42e0-a0e4-c2bd2181fda5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 808cb47b-1f31-491c-9c00-64e0ad0e5565 kind: Event lastTimestamp: "2026-09-18T16:35:30Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:35:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:30Z" name: isvc-canary-rollback-predictor-7f7b644c9-wsgs4.18d677888496695b namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: de7cb36c-d97f-402b-a5da-e58736b08e3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-7f7b644c9 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 66d17240-4bfa-473b-989a-7d217477de26 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: 'Created pod: isvc-canary-rollback-predictor-7f7b644c9-sxhhv' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-predictor-7f7b644c9.18d67759297b0aa2 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 0436f38c-8bb4-433a-9790-836bf52db5f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-7f7b644c9 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 66d17240-4bfa-473b-989a-7d217477de26 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: 'Created pod: isvc-canary-rollback-predictor-7f7b644c9-rdm8j' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-predictor-7f7b644c9.18d677592ef28d4b namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 8c2df125-491b-4668-b98d-0107528c4314 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-7f7b644c9 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 66d17240-4bfa-473b-989a-7d217477de26 kind: Event lastTimestamp: "2026-09-18T16:33:48Z" message: 'Deleted pod: isvc-canary-rollback-predictor-7f7b644c9-rdm8j' metadata: creationTimestamp: "2026-09-18T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:48Z" name: isvc-canary-rollback-predictor-7f7b644c9.18d67770d8d72c8f namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: bd521f7e-0fa2-43e7-a160-535e7ce49467 reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-predictor-7f7b644c9 namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: 66d17240-4bfa-473b-989a-7d217477de26 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: 'Created pod: isvc-canary-rollback-predictor-7f7b644c9-wsgs4' metadata: creationTimestamp: "2026-09-18T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback-predictor-7f7b644c9.18d67772c6c7fd8d namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 09496e3a-9cb9-4337-901b-727f7bc8496e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-predictor namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 05928558-4bb0-431f-88c5-320d364e60b3 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: Scaled up replica set isvc-canary-rollback-predictor-7f7b644c9 from 0 to 2 metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-predictor.18d6775925019217 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 3003948b-1cbb-4a82-8242-6e9117ca330d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-predictor namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: 05928558-4bb0-431f-88c5-320d364e60b3 kind: Event lastTimestamp: "2026-09-18T16:33:48Z" message: Scaled down replica set isvc-canary-rollback-predictor-7f7b644c9 from 2 to 1 metadata: creationTimestamp: "2026-09-18T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:48Z" name: isvc-canary-rollback-predictor.18d67770d85a9c11 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: d33dff4a-3bfc-4987-b71f-7a2c257ffc8d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-predictor namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 05928558-4bb0-431f-88c5-320d364e60b3 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: Scaled up replica set isvc-canary-rollback-predictor-7f7b644c9 from 1 to 2 metadata: creationTimestamp: "2026-09-18T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback-predictor.18d67772c01bc5c4 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: 1b0955da-a42d-4901-a982-7d0dcfb5d5e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:32:06.744574Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775928ad2c2c namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 4afb5dc4-bed0-42d2-a339-cca5c6121d2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-rollback-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d67759349d6fcb namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 0c6e44d1-0c13-43ed-ac1a-decce9cd9e54 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d677596e92f4e7 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: f501c5f9-92e0-48b3-b211-75847ac20e4f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:07Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d67759701f3474 namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 902bfbe6-ce01-407f-9db1-7c89ab7fc146 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:08Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775974118edc namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 75ec72ca-6974-4d0e-a1b8-430b5530d8c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:08Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775974843cbe namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 278132e6-3ef3-43ec-884f-28da7ab7b69a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:12Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775a6eb415bf namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: c3c5254e-c3bc-488c-8a1b-0ff6bdc2d160 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:28Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" in 16.753s (16.753s including waiting). Image size: 1607081368 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:28Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775e5549043e namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 71662180-446f-4953-baad-fdaa347bc1b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:29Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775e58205f7b namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 53f95346-e07c-4e87-bfed-a66b417071c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:29Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775e5889252b namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: c3fef442-fc70-4884-9e38-4706eae58de2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:29Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775e58983e07 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 36c1dba9-b980-4748-94b2-74fafd02b3e1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.395s (2.395s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:31Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775ee76627e7 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 30501135-706e-41be-8645-eeb0f5bcbe2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:31Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775eeae6d58d namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: f759d071-cbb3-44c0-a11f-55c0f25eeb44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:32:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:31Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775eeb4d72a4 namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: 519ba454-b411-4afd-b7b7-c3e0138a3ae6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:33:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:58Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6775f1eb19a12 namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: 6ad8634e-e921-492b-b0e7-c158ae2ff857 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d67772c16d290b namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: 8919f80a-b49d-4730-88ba-766080941f49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d67772c16e5127 namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: 1bb803f0-f465-4cbd-9019-5d04b5275d50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: a799c2d6-f36f-41a9-9104-db5ed9e102c2 kind: Event lastTimestamp: "2026-09-18T16:33:58Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:33:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:33:58Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf.18d6777325058e6f namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 8ba84e0f-1005-4471-be75-95b3c084c640 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-rollback-v2-predictor-56575f6cc4 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: a665b760-153e-47e2-905e-396fc5c296e3 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: 'Created pod: isvc-canary-rollback-v2-predictor-56575f6cc4-nzrtf' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-v2-predictor-56575f6cc4.18d677592822497b namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 93f9eee5-4cbf-401b-b25b-75328b4a876d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-rollback-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 6cf2ee39-df8e-492a-afa2-6286b0fdaaf6 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: Scaled up replica set isvc-canary-rollback-v2-predictor-56575f6cc4 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-canary-rollback-v2-predictor.18d677591dd99b2f namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: c5f4a5ff-0445-44bb-aa60-c039d110c87a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 20 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: ae3f7427-a7bf-487d-875e-3222afba62e5 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback.18d6775924a5b69f namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 3da3e44d-8634-4bc4-ad6a-3d3413ffb659 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: ae3f7427-a7bf-487d-875e-3222afba62e5 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: 'Failed to update status for InferenceService "isvc-canary-rollback": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback.18d677592536a7f1 namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: dfc8a4b8-4d12-4f97-9a01-e380ff37efd6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: ae3f7427-a7bf-487d-875e-3222afba62e5 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback.18d677592536eb96 namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 0c19469c-4e2d-4e8e-803f-9227635c23f3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:33:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: ae3f7427-a7bf-487d-875e-3222afba62e5 kind: Event lastTimestamp: "2026-09-18T16:35:20Z" message: InferenceService [isvc-canary-rollback] is Ready metadata: creationTimestamp: "2026-09-18T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:20Z" name: isvc-canary-rollback.18d67770dc35c595 namespace: kserve-ci-e2e-test resourceVersion: "18109" uid: 514198a2-aeda-4e39-a54b-db56167ef813 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-rollback namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: ae3f7427-a7bf-487d-875e-3222afba62e5 kind: Event lastTimestamp: "2026-09-18T16:33:56Z" message: 'InferenceService [isvc-canary-rollback] is no longer Ready because of: PredictorReady, Ready, Stopped' metadata: creationTimestamp: "2026-09-18T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:33:56Z" name: isvc-canary-rollback.18d67772ca5c751c namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: 5f4ea0ea-e1bd-4ee3-935f-76ee819aac02 reason: InferenceServiceNotReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:35:32.013962Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18264" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-5ffdccb94c-q8mwv to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d67788f3af759e namespace: kserve-ci-e2e-test resourceVersion: "18268" uid: b8d81b79-30fd-413e-84f6-affbbafc6432 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18272" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d67789159f5e99 namespace: kserve-ci-e2e-test resourceVersion: "18303" uid: cf642ebf-e060-4b7b-b7be-c3806082a2d5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d67789175a235c namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: 8541fead-d5b2-428e-824f-b55e6ad52787 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d677891a7f3ab6 namespace: kserve-ci-e2e-test resourceVersion: "18311" uid: d8f9401d-66d5-4353-a6f5-245dc27d96be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d677891ae457e5 namespace: kserve-ci-e2e-test resourceVersion: "18312" uid: e21bebf6-fdf6-484d-8511-565390b512be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:37Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a25990d6e namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: bb7eb416-8e00-4a91-84b2-4611c67a7993 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:37Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a28be9677 namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: b084a3cc-6bae-47ab-b196-651e254176a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:37Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a2921168d namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: 01331887-4f4a-4a49-ae62-e2e10fad4623 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:37Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a292b2728 namespace: kserve-ci-e2e-test resourceVersion: "18395" uid: 58e96c3c-e693-49c1-be0f-32290cb25729 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:37Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a2bce16bc namespace: kserve-ci-e2e-test resourceVersion: "18397" uid: 510fecd8-d23d-4919-b0aa-9d9070ab81c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:35:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:37Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a2c2980b4 namespace: kserve-ci-e2e-test resourceVersion: "18399" uid: 4c265fed-32fd-4eb9-a92a-cb251e50978c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:35:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:37:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:15Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d6778a9d788514 namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: 4484ca01-8d17-4c46-82c3-9bb6fb573d91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:37:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:11Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d677a03ac45638 namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: 00a41731-2317-4a60-b5ad-944d2f9f4f47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:37:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:11Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d677a03ac4f499 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 409f900a-667c-4c83-b0f8-dec4c8d0a723 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 03ed76b6-52be-4ecd-bbb3-755f5627619f kind: Event lastTimestamp: "2026-09-18T16:37:15Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:15Z" name: isvc-canary-stop-predictor-5ffdccb94c-q8mwv.18d677a0f7a8c528 namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: 582e994a-d77a-4938-977f-9f14df281a4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:35:31.963058Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18251" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-5ffdccb94c-zpw8r to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d67788f0a6bf10 namespace: kserve-ci-e2e-test resourceVersion: "18256" uid: 24657cd1-5553-490d-8e45-80d3caa3d451 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18263" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778913368977 namespace: kserve-ci-e2e-test resourceVersion: "18299" uid: f379e22f-9b3f-4b22-a120-158bcc0c8fc2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778914d2795d namespace: kserve-ci-e2e-test resourceVersion: "18302" uid: 68cc9fdb-4a69-4831-a1bc-92e0dc5c14b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d67789186e0c09 namespace: kserve-ci-e2e-test resourceVersion: "18307" uid: 3d818cea-2147-424a-901f-a35d0f2321e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d677891915f9b0 namespace: kserve-ci-e2e-test resourceVersion: "18310" uid: 9dcd2b6f-fe95-4978-b7f4-97c4765f1ba7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a014a0052 namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: 02fca310-3e66-4fe2-868d-90988cc123fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a04d22346 namespace: kserve-ci-e2e-test resourceVersion: "18375" uid: 4eeec99d-da59-4ad4-8a91-4cce189a835f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a054e13d4 namespace: kserve-ci-e2e-test resourceVersion: "18377" uid: 65d604f1-5e74-45c0-8f20-014cca3b8dbe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a05585705 namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: 5f388606-2a20-4dfd-be8f-89541347d21e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a09188fda namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: 55f7e4e8-dad0-48a8-ae1c-b28cdec319fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a09923454 namespace: kserve-ci-e2e-test resourceVersion: "18385" uid: a009c7e5-049d-4b11-8bcd-a2610440b453 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:36:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:56Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6778a7961b7f0 namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: f44326eb-66d1-468e-98d2-deae00b95750 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:37:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:09Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6779f9cc46223 namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: b935a9d5-7bf5-4041-9f1c-9cfbc7cc904c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:37:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:09Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6779f9cc4f8a3 namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: bccd4971-3e65-4433-8384-0e7af95b5cdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a4f41ccb-5c02-437e-b3c5-c93205cf3c7d kind: Event lastTimestamp: "2026-09-18T16:37:14Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:14Z" name: isvc-canary-stop-predictor-5ffdccb94c-zpw8r.18d6779fa99688ee namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: 3528f9f1-469f-42fe-8cce-40a615548b8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-predictor-5ffdccb94c namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: 7aaae3ae-fe8c-4c39-acc3-2d2d96654ffe kind: Event lastTimestamp: "2026-09-18T16:35:31Z" message: 'Created pod: isvc-canary-stop-predictor-5ffdccb94c-zpw8r' metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop-predictor-5ffdccb94c.18d67788f0306133 namespace: kserve-ci-e2e-test resourceVersion: "18252" uid: 72654834-9317-44ba-be38-0e41dd8e49d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-predictor-5ffdccb94c namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: 7aaae3ae-fe8c-4c39-acc3-2d2d96654ffe kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: 'Created pod: isvc-canary-stop-predictor-5ffdccb94c-q8mwv' metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-predictor-5ffdccb94c.18d67788f33b9b66 namespace: kserve-ci-e2e-test resourceVersion: "18265" uid: fbda12b8-3050-4a8e-8376-3d09543921d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-predictor-5ffdccb94c namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: 7aaae3ae-fe8c-4c39-acc3-2d2d96654ffe kind: Event lastTimestamp: "2026-09-18T16:37:09Z" message: 'Deleted pod: isvc-canary-stop-predictor-5ffdccb94c-zpw8r' metadata: creationTimestamp: "2026-09-18T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:09Z" name: isvc-canary-stop-predictor-5ffdccb94c.18d6779f9cb079bc namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 10c8603b-a466-4693-abf6-4326ce0f49c2 reason: SuccessfulDelete reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-stop-predictor namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: 6c241695-1787-4900-a31c-642d3af13f69 kind: Event lastTimestamp: "2026-09-18T16:35:31Z" message: Scaled up replica set isvc-canary-stop-predictor-5ffdccb94c from 0 to 2 metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop-predictor.18d67788ed55acb4 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: 81b88202-2f65-4025-b0bf-91785dbe5b10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-stop-predictor namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: 6c241695-1787-4900-a31c-642d3af13f69 kind: Event lastTimestamp: "2026-09-18T16:37:09Z" message: Scaled down replica set isvc-canary-stop-predictor-5ffdccb94c from 2 to 1 metadata: creationTimestamp: "2026-09-18T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:09Z" name: isvc-canary-stop-predictor.18d6779f9c316262 namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: fac2af45-4ecc-4358-9bb3-a87df766eb17 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:35:31.953037Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18247" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d67788f00dd4e6 namespace: kserve-ci-e2e-test resourceVersion: "18253" uid: 91e7e4e8-b9e7-4ce0-9539-aa23235d4b6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18262" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778912429903 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: 542db9da-63ff-437b-a68b-1de904cd99e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778913ddf5b2 namespace: kserve-ci-e2e-test resourceVersion: "18300" uid: 1cd16231-f408-4bf3-b3cf-b4b66621f342 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d677891878355d namespace: kserve-ci-e2e-test resourceVersion: "18308" uid: ff9e43af-d99b-4a23-8c10-8eedc231a4dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:32Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778918ef1fb0 namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: c8df7431-418a-48aa-89e8-43b2487257df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a015e966e namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: a2566275-7efa-4919-9159-757883280a85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a04df3d89 namespace: kserve-ci-e2e-test resourceVersion: "18376" uid: 7ec0e7a3-491d-4841-8c96-785e5b8fc446 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a055c40f5 namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: 95679d03-88f7-4d2d-a835-ba4b72887a8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a05682c4d namespace: kserve-ci-e2e-test resourceVersion: "18381" uid: 1aaaed26-c663-4bff-a58d-b1f43e8ea747 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a09247b59 namespace: kserve-ci-e2e-test resourceVersion: "18384" uid: 81015112-e4b3-42ad-a8fc-09895f49d33d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:35:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:36Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a099f3275 namespace: kserve-ci-e2e-test resourceVersion: "18386" uid: cbfaa9b0-8da3-45bf-80ad-162f3b80495b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:36:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:59Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d6778a3d9056f6 namespace: kserve-ci-e2e-test resourceVersion: "19269" uid: 85efea13-bdde-4048-9c80-2c2f485b8c9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:37:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:11Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d677a038821f46 namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 024c64b8-f174-4ed9-8ed7-f73f63b76c5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:37:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:11Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d677a0388353c0 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 78d9eea1-fe76-4709-ad2f-d1678e849bcb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: 3c657f83-3411-489e-a770-b0713ed17508 kind: Event lastTimestamp: "2026-09-18T16:37:13Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:13Z" name: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7.18d677a097c3b3e2 namespace: kserve-ci-e2e-test resourceVersion: "19430" uid: 5ce7ec68-1c76-4d6f-b97c-57780d9fc638 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-canary-stop-v2-predictor-b9c8d9798 namespace: kserve-ci-e2e-test resourceVersion: "18231" uid: 3c25b284-eb64-4507-bd38-69b7dda7eba9 kind: Event lastTimestamp: "2026-09-18T16:35:31Z" message: 'Created pod: isvc-canary-stop-v2-predictor-b9c8d9798-p7bh7' metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop-v2-predictor-b9c8d9798.18d67788ef8bbd55 namespace: kserve-ci-e2e-test resourceVersion: "18248" uid: 0befb447-21ac-45aa-969f-b803591922fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-canary-stop-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18230" uid: 2bb263fc-49aa-4b43-9bd6-993f80c8949a kind: Event lastTimestamp: "2026-09-18T16:35:31Z" message: Scaled up replica set isvc-canary-stop-v2-predictor-b9c8d9798 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop-v2-predictor.18d67788eb5d1435 namespace: kserve-ci-e2e-test resourceVersion: "18232" uid: 6e04a7c7-c00e-4b20-8c3e-f88d117c4bd7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 22 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "18222" uid: 8203f500-de7b-4039-a9d8-e21aa9979cf8 kind: Event lastTimestamp: "2026-09-18T16:37:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:11Z" name: isvc-canary-stop.18d67788ece3a929 namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 8e489d78-6101-4f8f-80a9-3333fd54efca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "18222" uid: 8203f500-de7b-4039-a9d8-e21aa9979cf8 kind: Event lastTimestamp: "2026-09-18T16:35:31Z" message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop.18d67788ed553c6b namespace: kserve-ci-e2e-test resourceVersion: "18242" uid: e46f7861-41f6-49e3-9537-7a0b285d3c55 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "18222" uid: 8203f500-de7b-4039-a9d8-e21aa9979cf8 kind: Event lastTimestamp: "2026-09-18T16:35:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:31Z" name: isvc-canary-stop.18d67788ed55848c namespace: kserve-ci-e2e-test resourceVersion: "18245" uid: 4dcb12ca-254d-46c4-b669-d1c91cd5cc33 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: 8203f500-de7b-4039-a9d8-e21aa9979cf8 kind: Event lastTimestamp: "2026-09-18T16:37:06Z" message: InferenceService [isvc-canary-stop] is Ready metadata: creationTimestamp: "2026-09-18T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:06Z" name: isvc-canary-stop.18d6779ee86f5130 namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: 1cccca5a-8312-4197-83fa-63615cd562e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-canary-stop namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 8203f500-de7b-4039-a9d8-e21aa9979cf8 kind: Event lastTimestamp: "2026-09-18T16:37:11Z" message: 'InferenceService [isvc-canary-stop] is no longer Ready because of: PredictorReady, Ready' metadata: creationTimestamp: "2026-09-18T16:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:11Z" name: isvc-canary-stop.18d677a03a71d5ad namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 53b5e31a-4515-4905-bc50-7e9106e007cb reason: InferenceServiceNotReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:30.712048Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-9d8350-predictor-8498856cb8-c89fx to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:30Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c08733345b namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: a8d84afa-95eb-461a-a758-f5eddf21ca65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: "2026-09-18T16:39:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-9d8350-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:30Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c09248c1dd namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: 2f7aa6d5-3617-429f-ab19-6f9334af571d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: "2026-09-18T16:39:31Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:31Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c0cd13bcff namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: d911da13-5117-4417-9b29-61965544bcb5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: "2026-09-18T16:39:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:36Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c0ceb756b9 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 2788e39d-4e24-4dfb-8b38-80383d4c76ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: "2026-09-18T16:39:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:36Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c0d1d78a88 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: e5bc9095-94e6-41ed-89dd-9af78f9a2c7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: "2026-09-18T16:39:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:36Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c0d24cb64f namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: b6a2ec96-220d-410c-8818-c76a585adff2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 2ff0983a-a461-4ce9-bd7d-8bfee00f99da kind: Event lastTimestamp: "2026-09-18T16:39:40Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-09-18T16:39:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:40Z" name: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx.18d677c2ddc024bc namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d31c0190-b7a4-40fb-936e-561e9ed33fce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-9d8350-predictor-8498856cb8 namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: c2c37048-9c5a-4b35-9078-2e2193dc5ad3 kind: Event lastTimestamp: "2026-09-18T16:39:30Z" message: 'Created pod: isvc-init-fail-9d8350-predictor-8498856cb8-c89fx' metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:30Z" name: isvc-init-fail-9d8350-predictor-8498856cb8.18d677c086c5066e namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 4cff0107-6d77-45cd-aa54-6e686da2e833 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-9d8350-predictor namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: ab8f4ca2-23b4-45a6-b9de-ef2a4297d33e kind: Event lastTimestamp: "2026-09-18T16:39:30Z" message: Scaled up replica set isvc-init-fail-9d8350-predictor-8498856cb8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:30Z" name: isvc-init-fail-9d8350-predictor.18d677c08413aad5 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: ebdade5f-fae0-4acf-ab25-203e41ee5f58 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T16:39:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-9d8350 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 754b4fc8-fe6f-4ac1-9f07-ea3b9e6f02e2 kind: Event lastTimestamp: "2026-09-18T16:39:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:36Z" name: isvc-init-fail-9d8350.18d677c0842240f6 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 1c7ef0b9-eab0-4417-8261-0b539008477c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-9d8350 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 754b4fc8-fe6f-4ac1-9f07-ea3b9e6f02e2 kind: Event lastTimestamp: "2026-09-18T16:39:30Z" message: 'Failed to update status for InferenceService "isvc-init-fail-9d8350": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-9d8350": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:30Z" name: isvc-init-fail-9d8350.18d677c084ab1ecf namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: a12fa124-e922-4cfc-8031-182a8c26d0a4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-9d8350 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 754b4fc8-fe6f-4ac1-9f07-ea3b9e6f02e2 kind: Event lastTimestamp: "2026-09-18T16:39:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-9d8350": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:30Z" name: isvc-init-fail-9d8350.18d677c084ab5471 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: d0c2c53a-5de5-4a76-8a75-55549f3d2555 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:38:22.917563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-031b1-predictor-7f898786d6-ff67l to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:38:22Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b0be55700f namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 0a46a88b-cc2d-49f0-a61b-2075199f66d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:23Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:38:23Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b0e02e8be7 namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: c4a8198d-d004-4057-8655-956f2fe9082e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:23Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b0e196b773 namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: 126b2697-7bc6-41ea-9777-cb99f1790973 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:23Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b0e400beb9 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 96145f7f-0604-4556-abf4-6b5cba9fa4a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:23Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b0e4ae5f0a namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: fca9954b-72cb-4093-b347-d5ed13d69acf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b201abef61 namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 758b119d-3aad-47d6-84c0-9999e8672db4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b204f7bbe0 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 571abc75-fdcf-4525-ac0c-1c5d425e440a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b205637401 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 323387f1-d935-4066-912a-62648a14f45a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b2056e773c namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f06d5ade-6bf4-44e3-b7ab-15d4c0407db6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b208610408 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 4cfc51f8-3133-4392-8436-34c0d338d27d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b208c9ddfe namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 01355944-5ccf-44c9-a53b-077bb0273287 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:bfbc00ba36039801e3cafbd0af9ffb0a52a9c8640f7ff416612ef57190d08496" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b208d37691 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 86838818-5a63-4c79-8efd-1dc9c4e67f13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b20be8d4ff namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 72fb01de-846a-4317-8582-cfe658a5a74c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:38:28Z" message: Started container agent metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:28Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b20c4a1d34 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: ee92d991-34f0-45e4-a36c-6db032c82b61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:38:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:40:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:06Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b23db13c81 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 69bfa0c3-ee3e-4b78-8a38-b32cc086c949 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:38:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:40:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:06Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677b279891111 namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 14705953-9b7c-4b37-847c-20c5e6ecb302 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:58Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677c706fb7091 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 2c495c3f-0e20-40d6-947a-3297312d1051 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:58Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677c706fbe087 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 5b56a1ec-ae2e-45ee-bc2d-9ebd1a3b2173 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:58Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677c706fc2290 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 9edfdbc9-a3d9-4a44-8843-ba05a3405435 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:40:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 3c8438bb-f470-4473-9b72-c46c989ac175 kind: Event lastTimestamp: "2026-09-18T16:40:16Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:16Z" name: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l.18d677c7a9c4f0d8 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: 3ce11827-d3da-40a2-afeb-ef5c2c4cfd1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-031b1-predictor-7f898786d6 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: e34d066d-ba12-4426-9432-a363d3bc04d7 kind: Event lastTimestamp: "2026-09-18T16:38:22Z" message: 'Created pod: isvc-logger-raw-031b1-predictor-7f898786d6-ff67l' metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:22Z" name: isvc-logger-raw-031b1-predictor-7f898786d6.18d677b0bdfde36c namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: ab791b37-f7da-4ec2-8689-5c8a2e516124 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: b8c0105d-be8e-40b9-806f-2652e18f926a kind: Event lastTimestamp: "2026-09-18T16:38:22Z" message: Scaled up replica set isvc-logger-raw-031b1-predictor-7f898786d6 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:22Z" name: isvc-logger-raw-031b1-predictor.18d677b0ba018426 namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: 89d92ff6-60e9-47af-a984-9293b1a44ec2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:38:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 28c0aa06-d8de-4c7b-b9b2-16c93f5e568a kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:07Z" name: isvc-logger-raw-031b1-predictor.18d677b438a52a3b namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 0b382924-d98a-41b0-82e8-3658688961cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:38:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 28c0aa06-d8de-4c7b-b9b2-16c93f5e568a kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:07Z" name: isvc-logger-raw-031b1-predictor.18d677b4391657fc namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 8f88d766-381c-4899-8454-71b06f9a23fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 28c0aa06-d8de-4c7b-b9b2-16c93f5e568a kind: Event lastTimestamp: "2026-09-18T16:39:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:52Z" name: isvc-logger-raw-031b1-predictor.18d677beb539eb6f namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: e3923aeb-1d5c-416c-870d-d3487ebd2096 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 28c0aa06-d8de-4c7b-b9b2-16c93f5e568a kind: Event lastTimestamp: "2026-09-18T16:39:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:52Z" name: isvc-logger-raw-031b1-predictor.18d677beb5b45c67 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: cb020fff-8b6f-4495-a39e-099f034eb9c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-031b1 namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: c11f7f4c-858b-41e1-ba6f-a03cc8bc2eb6 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:46Z" name: isvc-logger-raw-031b1.18d677b0ba07ca41 namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 577828a1-b369-4667-aaf6-3b9ae2538126 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-031b1 namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: c11f7f4c-858b-41e1-ba6f-a03cc8bc2eb6 kind: Event lastTimestamp: "2026-09-18T16:38:22Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-031b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-031b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:22Z" name: isvc-logger-raw-031b1.18d677b0baac56f4 namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: df2ce871-e538-400e-be20-09e4cfc5d481 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-031b1 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: c11f7f4c-858b-41e1-ba6f-a03cc8bc2eb6 kind: Event lastTimestamp: "2026-09-18T16:38:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-031b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:22Z" name: isvc-logger-raw-031b1.18d677b0c0e6423f namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: d14fdcfe-f34f-4742-ad40-d56de440a462 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-031b1 namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: c11f7f4c-858b-41e1-ba6f-a03cc8bc2eb6 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: InferenceService [isvc-logger-raw-031b1] is Ready metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:46Z" name: isvc-logger-raw-031b1.18d677c432a5ab54 namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 4ee39c44-9ea5-4bd4-be9b-1e9971611dbc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:48:07.598507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cipn-predictor-787489bd95-kqp6p to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:48:07Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d67838e008a1bd namespace: kserve-ci-e2e-test resourceVersion: "25807" uid: 7565fe21-1b09-4bbd-a0a9-560e64d1b356 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-cipn-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:07Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d67838e93072ac namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 2c50abe9-cec0-40aa-832f-d22c131c3108 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25811" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:08Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:48:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:48:08Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783925c58145 namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 7bd66cb5-4357-4280-9a60-6308a7085657 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:08Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d678392763b05b namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: 9b823e9e-0dd5-4b56-b2bf-71f48d83e76f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:48:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:08Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d678392b3ac58e namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: 72f2559a-bd7b-46ae-a164-b8df83c43026 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:48:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:08Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d678392baf2129 namespace: kserve-ci-e2e-test resourceVersion: "25833" uid: 5508d47b-bb84-48b1-870b-b897b59b4a41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a1a691b4d namespace: kserve-ci-e2e-test resourceVersion: "25866" uid: a9f1ea45-1430-488b-9640-b85916fff757 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a1d6c336d namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: 99c65251-78fe-434d-9d21-5e584cd10415 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a1de3b33d namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: 005268ea-247f-48cc-b568-2982970f7e33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a1def3b41 namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: a8bb0356-d3ea-41a3-bd56-f9efac2abaa0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a20d11efa namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: e5a74930-335a-4306-bb5c-15dc425b98f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a214987d8 namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 91b62fbb-3672-4ae5-bb8d-240f0d2f2e25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:12Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:bfbc00ba36039801e3cafbd0af9ffb0a52a9c8640f7ff416612ef57190d08496" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:12Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a2153e80c namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 4335435f-f727-4288-814b-6cbe2e809c8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:13Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-09-18T16:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:13Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a24540483 namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: b433af2b-3653-4501-bb2b-4a864cfc89e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:48:13Z" message: Started container agent metadata: creationTimestamp: "2026-09-18T16:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:13Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a24c9998f namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 5f52d9de-cc93-4bc2-be46-ad5401932d09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:49:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:49Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a5683fdae namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: 1e2c6902-bbe6-4392-9d69-4fe321799910 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:49:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:49Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6783a568f930d namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: 3745d36a-a077-41a8-b045-d6240ab5627f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:49:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:43Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6784f2506605f namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: 0624caf0-a7d6-4f4d-84c3-43fc2f5deaa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:49:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:43Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6784f25079226 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 9d9770d9-6cb0-4e58-9422-d021956d8c11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:49:43Z" message: Stopping container agent metadata: creationTimestamp: "2026-09-18T16:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:43Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6784f2507a81e namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: d1d0b840-a80d-4ec7-90f9-adccd855685d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: a692a891-8165-486f-b355-6a53917b00e2 kind: Event lastTimestamp: "2026-09-18T16:49:59Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:59Z" name: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p.18d6784f86b37228 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: 671ac8a8-2ca7-4218-b5a8-88ef7e130d4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-cipn-predictor-787489bd95 namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: 2bd81413-b73a-4b2c-9441-92626f74288f kind: Event lastTimestamp: "2026-09-18T16:48:07Z" message: 'Created pod: isvc-logger-raw-cipn-predictor-787489bd95-kqp6p' metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:07Z" name: isvc-logger-raw-cipn-predictor-787489bd95.18d67838df721eb7 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: bbfbd9a1-060f-4925-a96b-6902e93508d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: 8b02b146-fccf-4575-98b1-532993c236d2 kind: Event lastTimestamp: "2026-09-18T16:48:07Z" message: Scaled up replica set isvc-logger-raw-cipn-predictor-787489bd95 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:07Z" name: isvc-logger-raw-cipn-predictor.18d67838dbaa3689 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: af6b4f28-f73f-456a-b71b-6bf43c736fd5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:48:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: c8980090-c09d-4e6c-8ef5-4a9ea82d9144 kind: Event lastTimestamp: "2026-09-18T16:48:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:52Z" name: isvc-logger-raw-cipn-predictor.18d6783c5ab7de16 namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 95ddc4f5-0bb6-4f09-ba9d-8f8dcf4a4934 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:48:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: c8980090-c09d-4e6c-8ef5-4a9ea82d9144 kind: Event lastTimestamp: "2026-09-18T16:48:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:52Z" name: isvc-logger-raw-cipn-predictor.18d6783c5b2eea4f namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: f2a5d02e-29b1-4429-85df-1d6045862968 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: c8980090-c09d-4e6c-8ef5-4a9ea82d9144 kind: Event lastTimestamp: "2026-09-18T16:49:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:37Z" name: isvc-logger-raw-cipn-predictor.18d67846d810e35f namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: caa3828c-0a77-4663-9091-6ad6a1c1de6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: c8980090-c09d-4e6c-8ef5-4a9ea82d9144 kind: Event lastTimestamp: "2026-09-18T16:49:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:37Z" name: isvc-logger-raw-cipn-predictor.18d67846d8636cb8 namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: b6e5875e-a8e4-4e58-86f8-2cdb50c8b20e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:48:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 7709b125-0c07-497c-b0c5-7a6a59e17a84 kind: Event lastTimestamp: "2026-09-18T16:49:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:30Z" name: isvc-logger-raw-cipn.18d67838dbdb53d7 namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: f1fa603b-2fac-4bc0-a70f-4141d62c1bd3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 7709b125-0c07-497c-b0c5-7a6a59e17a84 kind: Event lastTimestamp: "2026-09-18T16:48:07Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-cipn": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:48:07Z" name: isvc-logger-raw-cipn.18d67838dc3a9bfb namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: 93caef1d-1155-4b31-9a2c-7a221b2ee8ff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 7709b125-0c07-497c-b0c5-7a6a59e17a84 kind: Event lastTimestamp: "2026-09-18T16:48:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:48:07Z" name: isvc-logger-raw-cipn.18d67838e2a12a8b namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: 533eb969-392d-4f19-95ba-d303dfa95204 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: 7709b125-0c07-497c-b0c5-7a6a59e17a84 kind: Event lastTimestamp: "2026-09-18T16:49:30Z" message: InferenceService [isvc-logger-raw-cipn] is Ready metadata: creationTimestamp: "2026-09-18T16:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:30Z" name: isvc-logger-raw-cipn.18d6784c0fe0bab5 namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: 0b7f7230-4fd6-4e14-99e3-14c376c1607d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:37:22.251531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:37:22Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a29e5b5359 namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 06f2db61-edf9-441d-b12f-03e3062de964 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-4e8a5a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:22Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a2a7d1de28 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: b083dd7c-2926-4c21-a3c4-43542be95fd7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:23Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:37:23Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a2e3f4ac7b namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: f97bb2e2-3117-4d37-9552-95f01d092d73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:23Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a2e5638444 namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: 0d109682-7a85-4c33-9734-7e1546a1c3f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:23Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a2e807934c namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: ddcebfe3-d2c1-48df-9246-0a494ec8e46a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:23Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a2e86e6db2 namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 15b0fcfc-a9ca-40d7-be6b-92d146a385fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:27Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:27Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a3dd6252ad namespace: kserve-ci-e2e-test resourceVersion: "19603" uid: 46f8ccd1-9b8b-43a7-884d-a7c3efb1fa25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:27Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a3e0d25ba8 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: 8ee7f7db-71d5-4ef3-bcc0-b5a754842c29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:27Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a3e1443673 namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 8e793139-869c-4875-8275-293fb7506695 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:27Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a3e14e5d1a namespace: kserve-ci-e2e-test resourceVersion: "19609" uid: 90f40115-480d-4f14-b1b6-5ef365b3eaad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:27Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a3e433b72b namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: 8840aa29-2b6e-455c-a2c9-0b52e1be4b23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:37:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:27Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a3e4a23a09 namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 4c8cd6ba-d095-4414-8e40-9fd1b3ab388a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:39:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:25Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677a4552a5ccb namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 3798acea-6876-4246-aff7-10583635a2c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:39:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:25Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677bf50324c79 namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 40cae37f-0bf1-4375-a432-2c5a3d4c0f13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:39:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:25Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677bf5032e915 namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 285328c8-a910-462f-b656-f6e60bd571f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: af63257f-9ae7-477a-99c0-0e09227687f4 kind: Event lastTimestamp: "2026-09-18T16:39:25Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:25Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt.18d677bf5787b8f5 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: d99442e0-9639-4c68-8ffe-5948a6707059 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-4e8a5a-predictor-567f87f6cf namespace: kserve-ci-e2e-test resourceVersion: "19529" uid: 6e469408-a51f-4086-982f-1a798112825e kind: Event lastTimestamp: "2026-09-18T16:37:22Z" message: 'Created pod: isvc-primary-4e8a5a-predictor-567f87f6cf-vpnbt' metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:22Z" name: isvc-primary-4e8a5a-predictor-567f87f6cf.18d677a29df7b5e0 namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: 4e68d645-dcb2-4836-bf40-a5a1e056ade3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19527" uid: 6eb2a263-f1bd-4262-9fb7-102aed7c55bc kind: Event lastTimestamp: "2026-09-18T16:37:22Z" message: Scaled up replica set isvc-primary-4e8a5a-predictor-567f87f6cf from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:22Z" name: isvc-primary-4e8a5a-predictor.18d677a29b0aac15 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: 3874b46c-d1e2-4117-8872-0df65fd3009e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:37:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: fc6b2b5e-5b76-4ada-9801-e08c12f783be kind: Event lastTimestamp: "2026-09-18T16:38:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:07Z" name: isvc-primary-4e8a5a-predictor.18d677a619ea7fe5 namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: ccfe6e0e-47c4-4a45-8c23-586b45fd099c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:37:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: fc6b2b5e-5b76-4ada-9801-e08c12f783be kind: Event lastTimestamp: "2026-09-18T16:38:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:07Z" name: isvc-primary-4e8a5a-predictor.18d677a61a6537a4 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 9f2d4130-5734-4f67-aff1-31f5667c1ee2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19707" uid: fc6b2b5e-5b76-4ada-9801-e08c12f783be kind: Event lastTimestamp: "2026-09-18T16:39:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:22Z" name: isvc-primary-4e8a5a-predictor.18d677b0989476d9 namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: f6054de6-4c33-406a-863c-57a8d321cf2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:38:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19707" uid: fc6b2b5e-5b76-4ada-9801-e08c12f783be kind: Event lastTimestamp: "2026-09-18T16:39:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:22Z" name: isvc-primary-4e8a5a-predictor.18d677b098ffc0d4 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: 8dbda239-c812-45b3-be86-23878cc8e357 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:37:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: d1ce296f-2fa4-4bf3-9e69-356f498960d7 kind: Event lastTimestamp: "2026-09-18T16:38:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:45Z" name: isvc-primary-4e8a5a.18d677a29b0d5991 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 5b046369-42ee-415b-8991-df31f1241b9f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: d1ce296f-2fa4-4bf3-9e69-356f498960d7 kind: Event lastTimestamp: "2026-09-18T16:37:22Z" message: 'Failed to update status for InferenceService "isvc-primary-4e8a5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-4e8a5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:22Z" name: isvc-primary-4e8a5a.18d677a29b96b7e5 namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: 5edf75af-4011-4100-8175-f041369a0973 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: d1ce296f-2fa4-4bf3-9e69-356f498960d7 kind: Event lastTimestamp: "2026-09-18T16:37:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-4e8a5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:22Z" name: isvc-primary-4e8a5a.18d677a29b96ec46 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 6e59dabd-73ba-4854-984b-0206e9f2404c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: d1ce296f-2fa4-4bf3-9e69-356f498960d7 kind: Event lastTimestamp: "2026-09-18T16:38:45Z" message: InferenceService [isvc-primary-4e8a5a] is Ready metadata: creationTimestamp: "2026-09-18T16:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:45Z" name: isvc-primary-4e8a5a.18d677b60c7ea12a namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 55c5fd71-1aa4-4713-bceb-4586bcca2c86 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:32:06.945854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:32:06Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775934ac6a7e namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 836074e4-ca4f-482b-b23a-9537322d53cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:32:07Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775956d85241 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: ba9abc50-0ed4-4ea5-84b1-5101a6952dd3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:07Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" metadata: creationTimestamp: "2026-09-18T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:07Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d67759590d6339 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 49726f9a-725b-433d-9293-3870e0374c25 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" in 4.193s (4.193s including waiting). Image size: 308174268 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775a5303367f namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: d2117cd9-8de6-4ff3-95ee-250509e427d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775a5640f20b namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: a8ab5eb8-bc42-4d92-abc3-8ae1d3719e0f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:11Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775a56fb00d9 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: b4a1d04c-7c6e-4035-9d18-5ecbead81e68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:15Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:15Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775b3a1f7b71 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: bdafb9f0-7bb9-48c0-a377-bf1e6bc9bc96 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" in 16.807s (16.807s including waiting). Image size: 1607081368 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775f23f49c32 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 2551e8f9-f860-497a-90ca-bf5991dccc3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775f277d981f namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 9a649309-5432-4129-86b9-919affd317f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775f27f1d600 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 5373a649-c0d7-4c05-b2ca-172e21b5f636 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:32Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775f27feb40f namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 7cbe5cfd-a847-4291-81cb-2ac1a73dea63 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.603s (2.603s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775fc331d038 namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 2ae588fc-d1bb-4098-9175-a8d5d3399ebe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775fc794ce5e namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 11d21946-8a88-4e67-b95c-867a283cdabf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775fc8368e09 namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 53a261fa-b9fc-4af7-9d58-1caffd8a0c47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:35Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:bfbc00ba36039801e3cafbd0af9ffb0a52a9c8640f7ff416612ef57190d08496" metadata: creationTimestamp: "2026-09-18T16:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:35Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6775fc843efd8 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: ff9c1f9b-2274-40dc-b896-280b87b342ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:37Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:bfbc00ba36039801e3cafbd0af9ffb0a52a9c8640f7ff416612ef57190d08496" in 2.571s (2.571s including waiting). Image size: 240302177 bytes.' metadata: creationTimestamp: "2026-09-18T16:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:37Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d677606185c5c9 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 844a2ba8-031f-40e2-bd79-4b58641d760e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:37Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-09-18T16:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:37Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d677606519ce09 namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 5a986d26-f10d-4db4-bef8-88b4b43b90d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:32:37Z" message: Started container agent metadata: creationTimestamp: "2026-09-18T16:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:32:37Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6776065bfd3f2 namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 7459865b-3714-41ec-ba6d-35e1c7c286cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:32:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:34:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:14Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d677609dbf9ae0 namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: 5014e016-9681-45a4-b594-0c67e52dd9a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:32:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:34:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:14Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d677609dc9a9c7 namespace: kserve-ci-e2e-test resourceVersion: "17606" uid: 45b19d2a-3267-4ce8-8250-8b2e0b2f09ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d677740096218f namespace: kserve-ci-e2e-test resourceVersion: "17383" uid: a2daf9f4-01a3-4e15-86f6-7c5e83b60b60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Stopping container agent metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d6777400967683 namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: a8cce478-d3ee-4937-b77b-3215f1827213 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d677740096b4b2 namespace: kserve-ci-e2e-test resourceVersion: "17387" uid: 6ba4f833-c800-44a9-96c4-4e3ea76c1421 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:34:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: c3ec894f-26e8-4f3f-a3c7-339bb7bb07d5 kind: Event lastTimestamp: "2026-09-18T16:34:14Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:14Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4.18d67774a3f3dbfe namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: d0396eea-a10e-4836-bad8-53079aa5ed17 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: a4f0c532-e382-4ab1-bea1-f73413dfdaae kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: 'Created pod: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8-tf5m4' metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8.18d6775934058448 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: af73c376-6069-4a92-84bb-cb45b0b69431 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-3f0a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: f09217d5-6894-45d5-a4df-500a616030f1 kind: Event lastTimestamp: "2026-09-18T16:32:06Z" message: Scaled up replica set isvc-raw-sklearn-batcher-3f0a6-predictor-5c676957f8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:32:06Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor.18d677592ce25597 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: eb249ea3-3150-4b75-814d-95c63956c066 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:32:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f0a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 8f8e0776-3a8e-4665-be7c-26d5a7cb5e77 kind: Event lastTimestamp: "2026-09-18T16:33:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:06Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor.18d6775cad1ea9b0 namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: e501594a-0f3c-4161-8509-8a396187da53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:32:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f0a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 8f8e0776-3a8e-4665-be7c-26d5a7cb5e77 kind: Event lastTimestamp: "2026-09-18T16:33:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:06Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor.18d6775cada4ad6d namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 71eaced1-4041-4df8-8a64-ba2366990b75 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:33:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f0a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 8f8e0776-3a8e-4665-be7c-26d5a7cb5e77 kind: Event lastTimestamp: "2026-09-18T16:33:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:33:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:52Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor.18d6776aaaf62cfc namespace: kserve-ci-e2e-test resourceVersion: "17233" uid: f2493cac-0eb3-45e7-8fb9-8029ce7493ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:33:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f0a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 8f8e0776-3a8e-4665-be7c-26d5a7cb5e77 kind: Event lastTimestamp: "2026-09-18T16:33:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:33:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:33:52Z" name: isvc-raw-sklearn-batcher-3f0a6-predictor.18d6776aab867942 namespace: kserve-ci-e2e-test resourceVersion: "17234" uid: c0de99fa-af7c-409b-9204-9f81f9b5249c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:32:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3f0a6 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: ad025ff9-b1db-47c5-b4ce-10b7c6a221f3 kind: Event lastTimestamp: "2026-09-18T16:33:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:33:55Z" name: isvc-raw-sklearn-batcher-3f0a6.18d677592ce76bf8 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 147a93fa-a8e0-497d-9df5-7249ce54f206 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:33:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3f0a6 namespace: kserve-ci-e2e-test resourceVersion: "17261" uid: ad025ff9-b1db-47c5-b4ce-10b7c6a221f3 kind: Event lastTimestamp: "2026-09-18T16:33:54Z" message: InferenceService [isvc-raw-sklearn-batcher-3f0a6] is Ready metadata: creationTimestamp: "2026-09-18T16:33:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:33:54Z" name: isvc-raw-sklearn-batcher-3f0a6.18d677725c0713a5 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: f97145d4-5e62-4064-babf-ee20075a25d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:07.419002Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: 4a559517-4284-4880-b536-7b5729ec1729 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:07Z" name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl.18d677bb1ad37c5c namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 49701261-0f3d-4bc6-8efa-f5b9b6ad9451 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 4a559517-4284-4880-b536-7b5729ec1729 kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:07Z" name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl.18d677bb3cb2a2d3 namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 9236f96b-5180-44a4-9ae1-6e220709af87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 4a559517-4284-4880-b536-7b5729ec1729 kind: Event lastTimestamp: "2026-09-18T16:39:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:13Z" name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl.18d677bb3e118a99 namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: 4e1f4aa7-e4c1-4780-9397-0b50a1f1dace reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 4a559517-4284-4880-b536-7b5729ec1729 kind: Event lastTimestamp: "2026-09-18T16:39:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:13Z" name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl.18d677bb40dc1c76 namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 8c82d8c7-46dd-42cf-90f4-1a132c854648 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 4a559517-4284-4880-b536-7b5729ec1729 kind: Event lastTimestamp: "2026-09-18T16:39:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:13Z" name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl.18d677bb4163bd58 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 62343da7-d989-475b-bb49-117243ea6663 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 4a559517-4284-4880-b536-7b5729ec1729 kind: Event lastTimestamp: "2026-09-18T16:39:16Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl_kserve-ci-e2e-test(4a559517-4284-4880-b536-7b5729ec1729) metadata: creationTimestamp: "2026-09-18T16:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:16Z" name: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl.18d677bd3b7c86de namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: af6ab3b6-3133-479d-a285-27af8199d28d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-4e8a5a-predictor-7c5466964 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 39532a7e-f1a0-402e-8860-14b5b940454a kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: 'Created pod: isvc-secondary-4e8a5a-predictor-7c5466964-6bnbl' metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:07Z" name: isvc-secondary-4e8a5a-predictor-7c5466964.18d677bb1a4b4d94 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 6ef6d35b-5d7d-4825-bffd-7202274becd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 6ebed4f4-a50c-42c4-b0bc-f5e109d8de3f kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: Scaled up replica set isvc-secondary-4e8a5a-predictor-7c5466964 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:07Z" name: isvc-secondary-4e8a5a-predictor.18d677bb174adda8 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 0e4e4e46-9e1b-42d5-866a-f228b24e02c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 7b7a0b01-3b7d-48ef-9062-12d27c333887 kind: Event lastTimestamp: "2026-09-18T16:39:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:22Z" name: isvc-secondary-4e8a5a-predictor.18d677be95b1ddb9 namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 8bcda9d4-65ff-4ae1-b51c-4fab096ebe85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-4e8a5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 7b7a0b01-3b7d-48ef-9062-12d27c333887 kind: Event lastTimestamp: "2026-09-18T16:39:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:22Z" name: isvc-secondary-4e8a5a-predictor.18d677be961a8212 namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 99d2f34e-cf16-43f1-8ae2-c5b46329fa8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: a094b685-381a-48e3-813b-a1b68036cecf kind: Event lastTimestamp: "2026-09-18T16:39:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:16Z" name: isvc-secondary-4e8a5a.18d677bb174e1437 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: e41023cf-f1ef-4cbb-88af-df9ae8605edf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: a094b685-381a-48e3-813b-a1b68036cecf kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: 'Failed to update status for InferenceService "isvc-secondary-4e8a5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-4e8a5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:07Z" name: isvc-secondary-4e8a5a.18d677bb17c7472f namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: e8cb764e-00d3-4843-acab-0b7bbecff280 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-4e8a5a namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: a094b685-381a-48e3-813b-a1b68036cecf kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-4e8a5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:07Z" name: isvc-secondary-4e8a5a.18d677bb17c7939f namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 766ef978-086e-4be1-94e4-1014fda3da3a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:34:02.194833Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17412" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:34:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d677740a0c4329 namespace: kserve-ci-e2e-test resourceVersion: "17420" uid: 22f1d3e6-653d-4902-b341-27e0485b53a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-0a5a4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d6777414e81da7 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: c6b4fa2f-76d5-4f87-939e-cfa6326533fb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:34:03Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d677744fe1afe1 namespace: kserve-ci-e2e-test resourceVersion: "17477" uid: c043078b-3195-4d64-95d8-cdff64718d62 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:03Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d67774517f53ce namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 60f72478-891c-4317-8329-506d547cf00e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:03Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d67774544906d1 namespace: kserve-ci-e2e-test resourceVersion: "17480" uid: 23b23049-3caa-49a1-b5d8-da27c89f60c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:03Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d6777454bf0fd6 namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: 94a1ff92-5ce3-448b-9f3a-d14289613dd3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d6777554e3c9da namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 94460da8-be7b-4360-9dad-cd5c5d79bf85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d677755924870c namespace: kserve-ci-e2e-test resourceVersion: "17540" uid: a1265f61-49fe-46af-8f25-39bb1bdb982e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d6777559aeb1b1 namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: 47e6e9fa-f422-4536-be4a-bb965e538be7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d6777559b870d9 namespace: kserve-ci-e2e-test resourceVersion: "17542" uid: 4612bcd0-4462-4e3d-b5d6-62bbfd5fefdf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d677755d170d56 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 7e48e5b5-be05-44f6-8f90-42bb06bdfb76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d677755d8492c9 namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: 9587041b-6dce-4514-86d0-62023ca74aaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:34:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d67775cd1fb831 namespace: kserve-ci-e2e-test resourceVersion: "18771" uid: b418efbe-86a4-49cc-b25a-631a815303cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d677900542f583 namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8f452828-5742-45e4-8ba3-7fc75653bd4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d67790054354d5 namespace: kserve-ci-e2e-test resourceVersion: "18678" uid: c637b2e6-06aa-42ab-add0-998ee01be2d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 6b73b611-6608-4cb4-9d17-986e245e01ec kind: Event lastTimestamp: "2026-09-18T16:36:05Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:05Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl.18d67790cf78da10 namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: ad03f508-83f0-4980-bd6d-5e9533ec811f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d namespace: kserve-ci-e2e-test resourceVersion: "17400" uid: aed972d0-ae37-4d78-89bf-121200d9e281 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'Created pod: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d-92ffl' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d.18d67774090ae3ed namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 88e1aad1-ddc9-4d49-bfaf-cf7cdd732e41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: 781ec316-4a86-45bf-b678-b880dcc9d9f1 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Scaled up replica set isvc-sklearn-graph-raw-0a5a4-predictor-9d6d5f65d from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor.18d6777404eeb152 namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: f1ae5a77-e81c-4d09-989b-0223a62bb5db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:34:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 613f3831-cebf-4a25-9c40-ce6753343137 kind: Event lastTimestamp: "2026-09-18T16:34:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:47Z" name: isvc-sklearn-graph-raw-0a5a4-predictor.18d6777783998de8 namespace: kserve-ci-e2e-test resourceVersion: "17861" uid: 51ec0e21-c081-487a-8412-d2d08c00a8d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:34:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 613f3831-cebf-4a25-9c40-ce6753343137 kind: Event lastTimestamp: "2026-09-18T16:34:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:47Z" name: isvc-sklearn-graph-raw-0a5a4-predictor.18d67777840ba949 namespace: kserve-ci-e2e-test resourceVersion: "17862" uid: 886402a8-7131-4895-9034-0d4637ff4614 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:35:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 613f3831-cebf-4a25-9c40-ce6753343137 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor.18d677820270d86c namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: 5dbd336b-b625-4548-a6ff-ae932c92284f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:35:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 613f3831-cebf-4a25-9c40-ce6753343137 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-0a5a4-predictor.18d6778202e40a1d namespace: kserve-ci-e2e-test resourceVersion: "18648" uid: a30c16d9-6f6b-46ee-bae3-6ffd7dd13c88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 5a1a7d70-b9ed-490c-8986-0cc5b9743268 kind: Event lastTimestamp: "2026-09-18T16:35:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:33Z" name: isvc-sklearn-graph-raw-0a5a4.18d67774050afd45 namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: 3451e8d7-a90b-4751-a4bb-9df5b6d243ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 5a1a7d70-b9ed-490c-8986-0cc5b9743268 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-0a5a4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-0a5a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:34:02Z" name: isvc-sklearn-graph-raw-0a5a4.18d6777405c36b64 namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: 2d7fb2c4-0f84-4729-bd1a-37f948d0875a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: 5a1a7d70-b9ed-490c-8986-0cc5b9743268 kind: Event lastTimestamp: "2026-09-18T16:35:33Z" message: InferenceService [isvc-sklearn-graph-raw-0a5a4] is Ready metadata: creationTimestamp: "2026-09-18T16:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:33Z" name: isvc-sklearn-graph-raw-0a5a4.18d677895c9113e8 namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: a9aac713-99ff-4fbd-b26c-d039318ebe1b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:36:02.511582Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677900d7c343d namespace: kserve-ci-e2e-test resourceVersion: "18718" uid: dd8b75e7-2d14-4a38-9e5b-c28b1f3e47c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-c8b7d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677901ad15697 namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 255391c5-8da3-4a92-9091-712eab6c1dd2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18727" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:36:03Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d6779053c5284f namespace: kserve-ci-e2e-test resourceVersion: "18772" uid: f8a7972b-2a81-43f8-b15c-6bd714d1f5cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d67790552b6dd2 namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: 2220b99b-7f18-4e26-9e72-d382ded9c245 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d6779058c3229f namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: fdcf9678-9a4b-4e4a-a024-66e35eee0c14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d67790598797e4 namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: 3bcd1d39-3793-425b-84a1-4d1eb439ee6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:07Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:07Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d67791413ec0db namespace: kserve-ci-e2e-test resourceVersion: "18833" uid: 9e70edf5-a2c5-4b24-aec5-dc0e8a1b2529 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:07Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677914489b061 namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: 8233d588-0169-4702-a06d-3acc890036ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:07Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d6779144fd7cf3 namespace: kserve-ci-e2e-test resourceVersion: "18839" uid: f70f1be5-5591-4c8e-a82f-467a48f69e79 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:07Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d6779145075366 namespace: kserve-ci-e2e-test resourceVersion: "18840" uid: 15e8f4df-4d69-42a4-96b9-9c31b4d76379 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:07Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d6779148b6244f namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: 9daf115c-e7fd-4440-8ad3-2db5ed176b6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:36:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:07Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d67791492de1d3 namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: da9adb0e-25ff-4be4-9dc0-dd2405a36998 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677917d78722a namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: d6f48a71-d348-4816-b98f-b5102819d96d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677ac0940cb5a namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 44fcdbb7-d465-408c-bb0e-7daa485d64e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677ac0941785e namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 8e35b82e-c895-4d62-81d0-e15b15d1bffc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 5ebe377b-af8c-4bbf-8641-74af533ca5d5 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6.18d677ac7ffd2e3d namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 751a8491-f09a-4f44-a400-e86413dee5eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b namespace: kserve-ci-e2e-test resourceVersion: "18703" uid: 2b9384c1-9b64-4dc8-85d4-ea9277040833 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b-6s2z6' metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b.18d677900d4189a3 namespace: kserve-ci-e2e-test resourceVersion: "18716" uid: a37288ce-f49c-4f06-8898-72cb03ee13f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: 57b7411b-cd90-4ee6-a173-73af060f35d4 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-c8b7d-predictor-7c948fbf4b from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor.18d677900a2a350f namespace: kserve-ci-e2e-test resourceVersion: "18704" uid: 471f1329-d220-4c60-9764-ba9980e85742 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:36:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 72cc378a-4f0c-4e5c-a8e7-1b3555c22ec2 kind: Event lastTimestamp: "2026-09-18T16:36:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:36:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:47Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor.18d6779388d06613 namespace: kserve-ci-e2e-test resourceVersion: "19172" uid: 39f7a06e-351d-4f17-a94a-aeb80d890ff2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:36:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 72cc378a-4f0c-4e5c-a8e7-1b3555c22ec2 kind: Event lastTimestamp: "2026-09-18T16:36:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:36:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:47Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor.18d67793893ce35c namespace: kserve-ci-e2e-test resourceVersion: "19173" uid: 987fcee4-ab17-4719-8606-00db5458eb3d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:36:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: 72cc378a-4f0c-4e5c-a8e7-1b3555c22ec2 kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:36:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor.18d6779708fd05be namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 9ffcd0b8-e48f-46c7-8d03-99d63b7754ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:36:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: 72cc378a-4f0c-4e5c-a8e7-1b3555c22ec2 kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:36:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:02Z" name: isvc-sklearn-graph-raw-hpa-c8b7d-predictor.18d677970958d5cd namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: cbc1ce25-a607-4d91-8ab1-76a40e338ec8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c8b7d namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: f6518edd-80f3-474b-88ba-62da81b6da6a kind: Event lastTimestamp: "2026-09-18T16:37:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:34Z" name: isvc-sklearn-graph-raw-hpa-c8b7d.18d677900a300263 namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: 26fa5397-7018-4332-83f4-c0eab66d3ded reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c8b7d namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: f6518edd-80f3-474b-88ba-62da81b6da6a kind: Event lastTimestamp: "2026-09-18T16:37:34Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-c8b7d] is Ready metadata: creationTimestamp: "2026-09-18T16:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:34Z" name: isvc-sklearn-graph-raw-hpa-c8b7d.18d677a5898d3320 namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 522ce336-96ac-4c86-833e-82ad37dc08b8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:58.759098Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:58Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c70eef369d namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 228b0631-2d8d-49b9-a29f-0a9cd24a9a1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:39:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-036a5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:59Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c71829b5df namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: a022e286-b401-4add-a635-27d621c86ba7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:00Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:40:00Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c78a6d4e9f namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 6445623a-0f5f-44ce-9376-e7dc8aa8aacc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:00Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c78c1709a1 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: fdd55a6c-8923-4f46-8b7f-c9f198f14d82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:00Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c78ee91697 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: a1543c90-b4bf-44f8-a6ad-8c4574b101bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:00Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c78f5c3906 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: b8f68d44-6e99-4b4b-84b3-1b28a97d4aab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:05Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:05Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c890f24887 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 475a7b7b-ccdb-42fa-a995-b0b54e9d9e01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:05Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c894191178 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: a213c422-bd9f-48a4-91cb-58fdc24a23b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:05Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c8948f29a8 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 86b9b9e4-838f-4a17-8179-8ed815317e54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:05Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c89499ff52 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: e4bc5082-a500-4103-85fa-122fb7e53c8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:05Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c897db1bf7 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 12be48bf-c0aa-440b-99d6-b846be29efb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:40:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:05Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c898521dbd namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: a8e64bc4-0d72-4852-94d7-0341ff2bde89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-09-18T16:40:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:43:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:02Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677c908c2af8f namespace: kserve-ci-e2e-test resourceVersion: "22950" uid: d0ed6fb8-e1dd-42fb-b0ac-7103487b28ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:42:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:59Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677f137854d6d namespace: kserve-ci-e2e-test resourceVersion: "22918" uid: b54c2494-2470-4dca-8cb1-f541d493ce04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:42:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:59Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677f137866f1d namespace: kserve-ci-e2e-test resourceVersion: "22919" uid: 77ce8ab6-0e2a-429e-9e03-e5ebc5aadda6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 6e52ac96-fa9e-4b45-8301-60f253b3d3b5 kind: Event lastTimestamp: "2026-09-18T16:43:08Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:43:08Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f.18d677f2035a2391 namespace: kserve-ci-e2e-test resourceVersion: "23017" uid: e7d63f52-53d6-48f0-a4ea-5d0b656cb6d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 031255ea-e92b-48e7-a395-220e9414cd66 kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: 'Created pod: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96-rrz2f' metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:58Z" name: isvc-sklearn-scale-raw-036a5-predictor-74b5966d96.18d677c70e902128 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: bca1a8d0-20be-4e16-9d03-d4602098279c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-036a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21294" uid: 5008eb98-05b1-4d42-9526-5d8e5431af2c kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: Scaled up replica set isvc-sklearn-scale-raw-036a5-predictor-74b5966d96 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:58Z" name: isvc-sklearn-scale-raw-036a5-predictor.18d677c70bda4924 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 65ef5635-d6a6-4a5d-b686-cf01588da165 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-036a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: d6aa2b44-7c9d-4d0b-8b94-947f63aad779 kind: Event lastTimestamp: "2026-09-18T16:40:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:13Z" name: isvc-sklearn-scale-raw-036a5-predictor.18d677ca8a5c0a33 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 097ea63a-40db-46ad-a6d8-2fc9dfe28ae2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-036a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: d6aa2b44-7c9d-4d0b-8b94-947f63aad779 kind: Event lastTimestamp: "2026-09-18T16:40:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:13Z" name: isvc-sklearn-scale-raw-036a5-predictor.18d677ca8ac18105 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: c42fff3b-b460-46d5-9700-425249041bbc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-036a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: d6aa2b44-7c9d-4d0b-8b94-947f63aad779 kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:58Z" name: isvc-sklearn-scale-raw-036a5-predictor.18d677ce09f2e07a namespace: kserve-ci-e2e-test resourceVersion: "22888" uid: 4107e2f6-b824-4fa8-b111-59937a61a387 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-036a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: d6aa2b44-7c9d-4d0b-8b94-947f63aad779 kind: Event lastTimestamp: "2026-09-18T16:42:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:58Z" name: isvc-sklearn-scale-raw-036a5-predictor.18d677ce0a5aef5d namespace: kserve-ci-e2e-test resourceVersion: "22891" uid: 7284841c-2a0f-4b9c-b7cc-f6331a060d2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-036a5 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: a311127a-0cc8-48b1-8fee-b1063149884c kind: Event lastTimestamp: "2026-09-18T16:42:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:42:52Z" name: isvc-sklearn-scale-raw-036a5.18d677c70bdeac60 namespace: kserve-ci-e2e-test resourceVersion: "22753" uid: 32ed6783-33cd-444f-904b-21d8f88fa668 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-036a5 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: a311127a-0cc8-48b1-8fee-b1063149884c kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-036a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-036a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:58Z" name: isvc-sklearn-scale-raw-036a5.18d677c70f5d57d4 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 07e1b2af-1728-427b-b6d7-76fd4831cde0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-036a5 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: a311127a-0cc8-48b1-8fee-b1063149884c kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-036a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:58Z" name: isvc-sklearn-scale-raw-036a5.18d677c70f5d989e namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 7ef74f4c-cc81-4639-aecc-99c7da71bb43 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-036a5 namespace: kserve-ci-e2e-test resourceVersion: "22750" uid: a311127a-0cc8-48b1-8fee-b1063149884c kind: Event lastTimestamp: "2026-09-18T16:42:52Z" message: InferenceService [isvc-sklearn-scale-raw-036a5] is Ready metadata: creationTimestamp: "2026-09-18T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:42:52Z" name: isvc-sklearn-scale-raw-036a5.18d677ef92f46f8b namespace: kserve-ci-e2e-test resourceVersion: "22752" uid: d068774a-5d59-4782-a335-dae376fd7dbe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:34:02.389221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17447" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:34:02Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777415a275dc namespace: kserve-ci-e2e-test resourceVersion: "17453" uid: 9b54aeb0-5e35-4296-ab32-feca18add98e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-0a5a4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:02Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777420e56164 namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: 18c978db-6d15-4cce-b540-648e06f20657 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17459" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:34:03Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d677745b9ca932 namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: c795d29f-a4dd-48d5-98fa-2e21dcbba5ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:03Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d677745d31f688 namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: e06595f3-0397-4021-bcda-41fb7de6f2fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:03Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777460c11aad namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: f3c3bcdc-c62a-4d09-8bbd-c050d8208c14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:03Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d67774613321f6 namespace: kserve-ci-e2e-test resourceVersion: "17486" uid: 9fd4cbb1-7798-4700-9aec-fd639980f106 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:07Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-18T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:07Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d677755500fdf3 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 23005408-799a-4572-9813-6395bec4d6e2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:27Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.115s (20.115s including waiting). Image size: 1436472580 bytes.' metadata: creationTimestamp: "2026-09-18T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:27Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a03fa9574 namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: 4be24ca7-063c-4f59-a0ef-725eefb8a039 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:27Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a0720ff7a namespace: kserve-ci-e2e-test resourceVersion: "17708" uid: e8199d3d-20ec-45d9-9fdd-a4d03fdb34f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:27Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a07836fdf namespace: kserve-ci-e2e-test resourceVersion: "17709" uid: d334cb91-fe0e-4b0a-bd1f-d79015d12fd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:27Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a078e3894 namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: 86325bb1-6b17-4bca-aafd-0e8875ffabd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:27Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a0a460594 namespace: kserve-ci-e2e-test resourceVersion: "17711" uid: 54431edb-4490-4be8-b7da-2a4c6392b568 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:34:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:34:27Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a0aa52569 namespace: kserve-ci-e2e-test resourceVersion: "17712" uid: 072d4b00-3798-4c28-8103-ad3c7c6b35af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T16:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:36:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:05Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6777a7ab34996 namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 88651c5e-e298-43dd-bd68-c2fc36d6ba9f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6779007a4e622 namespace: kserve-ci-e2e-test resourceVersion: "18693" uid: ebc7f93d-0948-46f4-8412-9ac3d7f711fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d6779007a57377 namespace: kserve-ci-e2e-test resourceVersion: "18694" uid: 9f5a702a-ab3e-406f-9572-4f03cea45c97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 65f04b2a-0186-41a3-87d8-44be882ad432 kind: Event lastTimestamp: "2026-09-18T16:36:05Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:05Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn.18d67790d4e4ea17 namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: de6bf2d1-f4af-45fa-b9f2-2d727fcdb510 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: 5a6c3197-2f13-4b56-a229-c5e425a916ad kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: 'Created pod: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f-pbkcn' metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:02Z" name: isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f.18d677741463d346 namespace: kserve-ci-e2e-test resourceVersion: "17448" uid: 8d37026f-8dc3-4af8-819c-ae42e30323b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: d03a8057-27cf-45f0-a0a2-76f8e381cacb kind: Event lastTimestamp: "2026-09-18T16:34:02Z" message: Scaled up replica set isvc-xgboost-graph-raw-0a5a4-predictor-844d56bc6f from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:02Z" name: isvc-xgboost-graph-raw-0a5a4-predictor.18d677740f367ab8 namespace: kserve-ci-e2e-test resourceVersion: "17441" uid: 1de7f8c4-729f-4531-b801-b9ecc86b286e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:34:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 45f616aa-9731-424d-b603-7b8ec56c5e6d kind: Event lastTimestamp: "2026-09-18T16:34:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:47Z" name: isvc-xgboost-graph-raw-0a5a4-predictor.18d677778df08fa2 namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: ee987b47-fc09-4a65-b590-23d0ca78d803 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:34:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 45f616aa-9731-424d-b603-7b8ec56c5e6d kind: Event lastTimestamp: "2026-09-18T16:34:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:34:47Z" name: isvc-xgboost-graph-raw-0a5a4-predictor.18d677778e5f2968 namespace: kserve-ci-e2e-test resourceVersion: "17865" uid: de5f225c-69da-479e-a5c4-9c4ac72d751b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:35:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 45f616aa-9731-424d-b603-7b8ec56c5e6d kind: Event lastTimestamp: "2026-09-18T16:35:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:47Z" name: isvc-xgboost-graph-raw-0a5a4-predictor.18d677820acb86e3 namespace: kserve-ci-e2e-test resourceVersion: "18484" uid: f0d4e0c8-ff59-49ea-b3bd-b22a83ee8c56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:35:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0a5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 45f616aa-9731-424d-b603-7b8ec56c5e6d kind: Event lastTimestamp: "2026-09-18T16:35:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:47Z" name: isvc-xgboost-graph-raw-0a5a4-predictor.18d677820b45c05c namespace: kserve-ci-e2e-test resourceVersion: "18486" uid: 21c8af87-f094-4daa-87ec-625d7564ba15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:34:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: b52f9b4c-358c-46cd-8645-c184b16d0acd kind: Event lastTimestamp: "2026-09-18T16:35:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:35Z" name: isvc-xgboost-graph-raw-0a5a4.18d677740f7a59db namespace: kserve-ci-e2e-test resourceVersion: "18366" uid: f166643d-eea8-4431-b0cf-73a2471688ca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: b52f9b4c-358c-46cd-8645-c184b16d0acd kind: Event lastTimestamp: "2026-09-18T16:35:35Z" message: InferenceService [isvc-xgboost-graph-raw-0a5a4] is Ready metadata: creationTimestamp: "2026-09-18T16:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:35Z" name: isvc-xgboost-graph-raw-0a5a4.18d67789de242e65 namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: ada7ef0b-c5cb-4554-8330-829271c7bbcc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:36:02.603565Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18740" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:36:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d6779012f7c08d namespace: kserve-ci-e2e-test resourceVersion: "18745" uid: b169c44f-5649-4b83-a030-678de16c42cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-c8b7d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d6779020ded06d namespace: kserve-ci-e2e-test resourceVersion: "18762" uid: ef38f5ce-ccc6-40cd-b570-614a518a8adf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18751" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:36:03Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d6779058cfd011 namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 21ab613e-af5e-4752-99e9-94daaccdfd54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677905a484a8e namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: 3dde5fe9-bb16-431d-82a8-f64550d3804e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677905d8ad4a9 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 5ddbdd54-5cf1-4968-8a17-199e2c0ceee0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:03Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677905e04e262 namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: 35226057-e9d6-400f-a098-a3d6ab85cfa3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:08Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-18T16:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:08Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677917d480c05 namespace: kserve-ci-e2e-test resourceVersion: "18851" uid: 19908c2d-6dbb-4c47-80b0-92939a58a8b2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:28Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.493s (19.493s including waiting). Image size: 1436472580 bytes.' metadata: creationTimestamp: "2026-09-18T16:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:28Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677960729f5f7 namespace: kserve-ci-e2e-test resourceVersion: "19010" uid: ddcb5d7b-31f5-4aea-a624-a58de2c79092 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:28Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677960a44169e namespace: kserve-ci-e2e-test resourceVersion: "19011" uid: 6df5a5ae-f755-4620-b2e5-79345ea8d656 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:28Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677960ac8d209 namespace: kserve-ci-e2e-test resourceVersion: "19012" uid: 30672f45-730d-42e5-9b1e-6a716d03f50e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:28Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677960ad4203b namespace: kserve-ci-e2e-test resourceVersion: "19013" uid: e37763fe-9345-457f-a3c2-18d186dd1757 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:28Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677960df68039 namespace: kserve-ci-e2e-test resourceVersion: "19014" uid: ee6867e5-b75a-4a9b-8866-10f1a587c93f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:28Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677960ebb2361 namespace: kserve-ci-e2e-test resourceVersion: "19015" uid: 61598868-8c2f-4a0b-ae10-a9f2bcd83a09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T16:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:38:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:05Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d6779667f0ff12 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: 8be27618-ca44-4a8e-8223-504a9f80594b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677ac0b8e9389 namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: a3fb89ff-1acf-4a28-8c33-73e6b152657f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677ac0b8fcc9e namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: b5d608ef-cb45-48e0-a5de-883a086bd7a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: d569322a-afa8-40df-b5ba-fd4b9b8869bd kind: Event lastTimestamp: "2026-09-18T16:38:05Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:05Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd.18d677acc240f99f namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: d4ad929d-b57f-421a-af96-17e5061970e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889 namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: 016ec93e-04e9-4531-81f0-fc8a4e7c1fe4 kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889-hvqrd' metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889.18d6779012653356 namespace: kserve-ci-e2e-test resourceVersion: "18741" uid: 2b345323-773f-4d7c-b4fd-f112255cacda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18728" uid: c0e2b82f-9e57-4331-a9a5-25f50e30868b kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-c8b7d-predictor-57cd759889 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:36:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor.18d677900fea0b22 namespace: kserve-ci-e2e-test resourceVersion: "18735" uid: b7c783d0-08dc-4505-876d-ace305f613c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:36:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: be8cef85-1afa-49d0-b586-787cc353b980 kind: Event lastTimestamp: "2026-09-18T16:37:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:36:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor.18d677938e338901 namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: 9f76d25c-2eec-4254-a5d2-26cefbd0257a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:36:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: be8cef85-1afa-49d0-b586-787cc353b980 kind: Event lastTimestamp: "2026-09-18T16:37:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:36:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor.18d677938e9cedf7 namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 372e9e77-e06a-4034-9e16-d23c482628ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:37:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18926" uid: be8cef85-1afa-49d0-b586-787cc353b980 kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor.18d677a189db7fa8 namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 467692f0-4d82-45f2-9577-e135576fa966 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:37:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18926" uid: be8cef85-1afa-49d0-b586-787cc353b980 kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:02Z" name: isvc-xgboost-graph-raw-hpa-c8b7d-predictor.18d677a18a477aca namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 81b50a17-0529-40bf-8593-55f710c4bec6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c8b7d namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: be57b320-3318-4ee3-965a-cf5e95e18d44 kind: Event lastTimestamp: "2026-09-18T16:37:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:35Z" name: isvc-xgboost-graph-raw-hpa-c8b7d.18d677900fe9a7fc namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 6ebc22f1-85eb-45a6-ac50-c4bf534351ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c8b7d namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: be57b320-3318-4ee3-965a-cf5e95e18d44 kind: Event lastTimestamp: "2026-09-18T16:37:35Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-c8b7d] is Ready metadata: creationTimestamp: "2026-09-18T16:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:35Z" name: isvc-xgboost-graph-raw-hpa-c8b7d.18d677a5cb80491f namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: c357437d-688a-4ae3-9a33-a9dd95d76d68 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:38:02.816847Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-031b1-predictor-57d74fc557-tcw9m to ip-10-0-138-142.ec2.internal metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:38:02Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac103cdc43 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: f08c40a1-ae02-4656-9f6c-f7f7f0c5a14e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:03Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20012" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:03Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:38:03Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac32341d82 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: b4adb362-063b-4ccb-a183-06c7ee64da84 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:03Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-09-18T16:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:03Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac33b02e0e namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: 9282c242-9245-421f-93d2-402ee8a2ee3c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.423s (1.423s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac888d8bcd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: de439f5e-884e-4306-aeca-777db6254fdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac8b7b4db0 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 1c9d2f40-6d82-4d21-9faa-d645959695c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac8be9f467 namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: 8e2c6280-dcf4-4149-b763-e5efd8a83d35 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac8bf3c769 namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: a2dcbcc0-096b-4023-87c7-76e4d4370202 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac8ef4d6b2 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: d680e4de-2701-4f54-a34c-dc333dc264f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:38:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:04Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677ac8f621fd3 namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 43de99a2-8cb0-4c21-a63e-27e6d8b83178 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:58Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677c708dd3c89 namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 7d51c17a-c200-41ec-836a-cff9c8008394 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ab1b734f-69ba-40f0-b82e-c6a9a72c4711 kind: Event lastTimestamp: "2026-09-18T16:39:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:58Z" name: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m.18d677c708ddb0f6 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 18a9941e-1f56-40ec-b6a2-955e03c7b1ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-142.ec2.internal source: component: kubelet host: ip-10-0-138-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-031b1-predictor-57d74fc557 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: dbc54e62-01b3-425b-94de-8eac9d5b957a kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: 'Created pod: message-dumper-raw-031b1-predictor-57d74fc557-tcw9m' metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:02Z" name: message-dumper-raw-031b1-predictor-57d74fc557.18d677ac0fdaf011 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 2b9ad740-243f-426f-88a8-3d12b8e3c426 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 7be26b52-16cb-4590-b857-8e798635862d kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: Scaled up replica set message-dumper-raw-031b1-predictor-57d74fc557 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:02Z" name: message-dumper-raw-031b1-predictor.18d677ac0e34ee4f namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 6380cf69-f682-40a3-a168-703bc7d1fb8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 9b0bfe91-fec2-4886-bee2-45dbeac9132e kind: Event lastTimestamp: "2026-09-18T16:38:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:32Z" name: message-dumper-raw-031b1-predictor.18d677af8cb7bf2b namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 1b520da9-d459-4886-bdeb-8fcbc4bdd5d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-031b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 9b0bfe91-fec2-4886-bee2-45dbeac9132e kind: Event lastTimestamp: "2026-09-18T16:38:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:32Z" name: message-dumper-raw-031b1-predictor.18d677af8d1ba894 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 6822c653-3ed4-4c7d-809d-95d45495bb23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-031b1 namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 0e7c6038-5a49-438e-b8d9-5606b395e80a kind: Event lastTimestamp: "2026-09-18T16:38:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:12Z" name: message-dumper-raw-031b1.18d677ac0e3a96c7 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: f646a695-f4f6-4fa2-921a-85638e597ea1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-031b1 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: 0e7c6038-5a49-438e-b8d9-5606b395e80a kind: Event lastTimestamp: "2026-09-18T16:38:12Z" message: InferenceService [message-dumper-raw-031b1] is Ready metadata: creationTimestamp: "2026-09-18T16:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:12Z" name: message-dumper-raw-031b1.18d677ae66078528 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 41f59462-0cec-40d4-8fbe-3f3fea3ade8a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:47:57.555318Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25675" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cipn-predictor-9774bf55d-zmpql to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:47:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:47:57Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d678368969b3a9 namespace: kserve-ci-e2e-test resourceVersion: "25679" uid: 62e0d9bc-c390-458b-849c-b040a9ca667e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:58Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:58Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:47:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:47:58Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d67836ab45d6e8 namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 4d4c1df2-bc4f-46e5-ac79-aeb4582f4dd8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:58Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-09-18T16:47:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:58Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d67836ac90d2d9 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 500002b0-d0ec-4279-83ff-f2ce8eda93c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:59Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.399s (1.399s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-09-18T16:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:59Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d67837000326e9 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 79a27dc1-51fc-4850-a74b-9c7c724c7ad8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:59Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d6783703178c2f namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 582bda67-12cc-4cbb-9a7a-f80164dac4f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:59Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d678370391631c namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: e850feaf-b8f6-4d38-bada-edfa4bf14d29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:59Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d67837039c3342 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 83f6fe52-ffc8-49c0-93c7-f2611978ee61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:59Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d6783706a3d1c5 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: b32260ef-bfab-47a1-935c-1ac4b3d6ca07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:47:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:47:59Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d67837071bb0a9 namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: cb237bdc-f4e6-49ca-91b5-a1fae611f735 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:49:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:43Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d6784f26ac4f6f namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 8fba2f4d-f24b-4003-8db6-997d61b589b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 132b9401-18e2-4ed8-afef-702ab650d6ba kind: Event lastTimestamp: "2026-09-18T16:49:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:43Z" name: message-dumper-raw-cipn-predictor-9774bf55d-zmpql.18d6784f26ad733f namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: 771bcf34-c5d6-48b6-acd4-a1a262ebf960 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-cipn-predictor-9774bf55d namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 9b5c8867-a622-4046-9f9d-7e5500ab6be0 kind: Event lastTimestamp: "2026-09-18T16:47:57Z" message: 'Created pod: message-dumper-raw-cipn-predictor-9774bf55d-zmpql' metadata: creationTimestamp: "2026-09-18T16:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:47:57Z" name: message-dumper-raw-cipn-predictor-9774bf55d.18d678368902c124 namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: 67389488-97e7-462b-adc9-8fb8a416b618 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: 9100a83b-b854-4852-ac02-84153ae18e72 kind: Event lastTimestamp: "2026-09-18T16:47:57Z" message: Scaled up replica set message-dumper-raw-cipn-predictor-9774bf55d from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:47:57Z" name: message-dumper-raw-cipn-predictor.18d6783685a38dc0 namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: 2a6f3702-5214-432e-94cb-b9747b07e068 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: 21836ec1-61bc-4f34-8fb9-09b1fd482d93 kind: Event lastTimestamp: "2026-09-18T16:48:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:27Z" name: message-dumper-raw-cipn-predictor.18d6783a04e31827 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 019f0433-8233-4363-8091-6b76bb97cf3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cipn-predictor namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: 21836ec1-61bc-4f34-8fb9-09b1fd482d93 kind: Event lastTimestamp: "2026-09-18T16:48:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:27Z" name: message-dumper-raw-cipn-predictor.18d6783a054f589f namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: b8a70047-4204-4c6d-8193-5980c1217fd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:47:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 1b58db95-7da9-43ed-a2d2-ee0f14bf4469 kind: Event lastTimestamp: "2026-09-18T16:48:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:48:06Z" name: message-dumper-raw-cipn.18d6783685aee576 namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: 464e9a80-c736-47a7-83b6-539c9432f6ef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:47:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 1b58db95-7da9-43ed-a2d2-ee0f14bf4469 kind: Event lastTimestamp: "2026-09-18T16:47:57Z" message: 'Failed to update status for InferenceService "message-dumper-raw-cipn": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cipn": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:47:57Z" name: message-dumper-raw-cipn.18d67836862d37f3 namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: 1442b29b-dacb-4a56-bcf6-fc7354117f1a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cipn namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: 1b58db95-7da9-43ed-a2d2-ee0f14bf4469 kind: Event lastTimestamp: "2026-09-18T16:48:06Z" message: InferenceService [message-dumper-raw-cipn] is Ready metadata: creationTimestamp: "2026-09-18T16:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:48:06Z" name: message-dumper-raw-cipn.18d67838b6e82b5d namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: bca4e1dd-7365-493e-8cfd-34ffd246685b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:35:52.207190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-0a5a4-847cb7f868-z4j45 to ip-10-0-139-57.ec2.internal metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778da74ba82f namespace: kserve-ci-e2e-test resourceVersion: "18532" uid: 6e51a25e-4302-42cd-ab34-cda960179804 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:52Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:35:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-0a5a4-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778db4bbfaf6 namespace: kserve-ci-e2e-test resourceVersion: "18546" uid: 7fcd2c0b-05e8-4031-877a-1f72f5603cf2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:53Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18536" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:35:53Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:35:53Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778ded895bc6 namespace: kserve-ci-e2e-test resourceVersion: "18559" uid: bb3b0c23-bf41-4222-8734-8768baa53809 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0a5a4} kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:35:53Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" metadata: creationTimestamp: "2026-09-18T16:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:53Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778deee0e55c namespace: kserve-ci-e2e-test resourceVersion: "18561" uid: bb5289f8-b2c6-45af-aead-0e9f1d0c36e3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0a5a4} kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:35:55Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" in 2.38s (2.38s including waiting). Image size: 211751521 bytes.' metadata: creationTimestamp: "2026-09-18T16:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:55Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778e7ccb7d4e namespace: kserve-ci-e2e-test resourceVersion: "18581" uid: c81438df-2211-48a0-94e7-76ac1682287b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0a5a4} kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:35:55Z" message: 'Created container: model-chainer-raw-0a5a4' metadata: creationTimestamp: "2026-09-18T16:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:55Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778e7fd24496 namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: ef6cd45f-4b64-4cea-af6e-446b58cd08be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0a5a4} kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:35:55Z" message: Started container model-chainer-raw-0a5a4 metadata: creationTimestamp: "2026-09-18T16:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:35:55Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778e8056ecbd namespace: kserve-ci-e2e-test resourceVersion: "18584" uid: 47821c83-8606-4d02-a3f4-70bcede110ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0a5a4} kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:36:02Z" message: Stopping container model-chainer-raw-0a5a4 metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:02Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778fff93c8c3 namespace: kserve-ci-e2e-test resourceVersion: "18650" uid: 0eec4ba6-1e30-46b0-838e-9bd145395e16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0a5a4} kind: Pod name: model-chainer-raw-0a5a4-847cb7f868-z4j45 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: a4b8e9d8-a335-416e-b2e5-3a327c1867af kind: Event lastTimestamp: "2026-09-18T16:36:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:36:32Z" name: model-chainer-raw-0a5a4-847cb7f868-z4j45.18d6778fffe554c1 namespace: kserve-ci-e2e-test resourceVersion: "19045" uid: b0957376-18f5-4c57-a671-aabe9c6a57f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-0a5a4-847cb7f868 namespace: kserve-ci-e2e-test resourceVersion: "18524" uid: f8c46dc7-1318-4971-8a2c-7f8362e35f2d kind: Event lastTimestamp: "2026-09-18T16:35:52Z" message: 'Created pod: model-chainer-raw-0a5a4-847cb7f868-z4j45' metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4-847cb7f868.18d6778da70e0c6b namespace: kserve-ci-e2e-test resourceVersion: "18530" uid: c9122f5c-15fe-4236-bcd6-b217f7fab588 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "18522" uid: 4124b822-3b57-4709-a249-f5fe24120f30 kind: Event lastTimestamp: "2026-09-18T16:35:52Z" message: Scaled up replica set model-chainer-raw-0a5a4-847cb7f868 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4.18d6778da632e5ba namespace: kserve-ci-e2e-test resourceVersion: "18525" uid: 5cc7fb1a-ec4d-4592-b9d3-81a7018371d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "18538" uid: 048d94f8-6774-486a-92b8-5269e94a4b05 kind: Event lastTimestamp: "2026-09-18T16:35:52Z" message: InferenceGraph [model-chainer-raw-0a5a4] is Ready metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4.18d6778dac65b39d namespace: kserve-ci-e2e-test resourceVersion: "18540" uid: ad3c6e1b-e270-4b2d-8593-c439bb0dade0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "18516" uid: 048d94f8-6774-486a-92b8-5269e94a4b05 kind: Event lastTimestamp: "2026-09-18T16:35:52Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-0a5a4": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-0a5a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4.18d6778daeba09bb namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: 376579a8-cde1-41e2-9072-ff0a9fc19dbe reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:35:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "18516" uid: 048d94f8-6774-486a-92b8-5269e94a4b05 kind: Event lastTimestamp: "2026-09-18T16:35:52Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-0a5a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:35:52Z" name: model-chainer-raw-0a5a4.18d6778daeba56d5 namespace: kserve-ci-e2e-test resourceVersion: "18542" uid: d1f76086-5782-499e-ab06-f6b684e7f83d reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:37:52.546760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-c8b7d-5f69564887-z699v to ip-10-0-139-57.ec2.internal metadata: creationTimestamp: "2026-09-18T16:37:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:37:52Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677a9ac17c3cd namespace: kserve-ci-e2e-test resourceVersion: "19836" uid: 63beb7ab-8cd3-4f4a-9b88-aad13d9b6647 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:53Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: "2026-09-18T16:37:53Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:37:53Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677a9cdd634e2 namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: 45f11834-7473-4a85-9b12-f81b95297f53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c8b7d} kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: "2026-09-18T16:37:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:53Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677a9cf85c844 namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 714b5747-4d74-4a62-b350-f4ce8d2776f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c8b7d} kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: "2026-09-18T16:37:53Z" message: 'Created container: model-chainer-raw-hpa-c8b7d' metadata: creationTimestamp: "2026-09-18T16:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:53Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677a9d2978da0 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 8e0bdebf-96d8-4627-8579-c5a5d38d6a4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c8b7d} kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: "2026-09-18T16:37:53Z" message: Started container model-chainer-raw-hpa-c8b7d metadata: creationTimestamp: "2026-09-18T16:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:53Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677a9d304c958 namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: b505127b-b1a9-488e-a94a-16b0d6767fb4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c8b7d} kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: "2026-09-18T16:38:02Z" message: Stopping container model-chainer-raw-hpa-c8b7d metadata: creationTimestamp: "2026-09-18T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:02Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677ac0352b97d namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: bc6c501a-baf6-49d5-b4de-cd9d1bf7438e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:38:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c8b7d} kind: Pod name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: d70dc658-2b8f-4a54-a37e-93f6534e7eba kind: Event lastTimestamp: "2026-09-18T16:38:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:30Z" name: model-chainer-raw-hpa-c8b7d-5f69564887-z699v.18d677ac96f78558 namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 2d64872d-6c7c-41bf-a0f0-b80d90b44da0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-57.ec2.internal source: component: kubelet host: ip-10-0-139-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-c8b7d-5f69564887 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: ea4d06a7-be5d-4fdb-9dcd-e0ea116ce9c7 kind: Event lastTimestamp: "2026-09-18T16:37:52Z" message: 'Created pod: model-chainer-raw-hpa-c8b7d-5f69564887-z699v' metadata: creationTimestamp: "2026-09-18T16:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:52Z" name: model-chainer-raw-hpa-c8b7d-5f69564887.18d677a9ab8f758d namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: e707f0ec-150f-42f3-9772-332f1a656e92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-c8b7d namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: eb38362b-c07d-4d27-b2d3-548daff978a5 kind: Event lastTimestamp: "2026-09-18T16:37:52Z" message: Scaled up replica set model-chainer-raw-hpa-c8b7d-5f69564887 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:52Z" name: model-chainer-raw-hpa-c8b7d.18d677a9aa974d53 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 269670a0-da96-491c-a64f-b5c547f89700 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-c8b7d namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: de3643a3-194f-4f23-b24e-c84d6960a121 kind: Event lastTimestamp: "2026-09-18T16:37:52Z" message: InferenceGraph [model-chainer-raw-hpa-c8b7d] is Ready metadata: creationTimestamp: "2026-09-18T16:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:37:52Z" name: model-chainer-raw-hpa-c8b7d.18d677a9b124e8f8 namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 3366cfe9-1cb2-4906-b722-cf440a2dfb11 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:45.999153Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-723b8-predictor-66b5f9879d-nssdq to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:46Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c41661f74d namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 4edea37c-e2fa-4b86-a4f7-e8730d782fd3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:46Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c4389200e7 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 12d0d010-7111-4421-92c1-e4c22a52d5d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c439f71460 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 2bc1d93f-13d1-4aec-b5ee-dfea616b8058 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c43d098036 namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 46b961af-6e62-4c16-a435-7997a13c056f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c43d80cbe0 namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: d922602d-70df-42ad-bb21-c08bf8815dd1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:50Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c52f372de3 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 246aadc5-c07b-4a3d-94ee-98b982cac7dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:50Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c5326c8dcf namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 2434cf62-bae5-442f-a312-8f0c7f0e6fdb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:50Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c532e144e3 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 0788eca9-018b-4eac-9cb0-53b7454f037a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:50Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c532ece62a namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: de662d89-73fb-4189-9ce0-26263019076d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:50Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c535b7fbac namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: e16f78b7-0702-43df-8eac-c5532d7de544 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:39:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:50Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c53629f710 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: f85daea0-dd41-4114-bebe-5e8ed2a6f35f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:41:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:18Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677c5a7178e41 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: df68971e-f530-4441-b859-014860be06a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:41:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:41:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:16Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677d928e93880 namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: b7d8d5bc-c46f-4f8f-893d-42504ac7357d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:41:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:41:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:16Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677d928e9e2f0 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 50c841de-d0d2-404f-bf8a-84baf2d218dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 4eba24bc-a9a7-416e-93f6-d623ae0ae64b kind: Event lastTimestamp: "2026-09-18T16:41:18Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:18Z" name: raw-sklearn-723b8-predictor-66b5f9879d-nssdq.18d677d9ad5438a1 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: c0485ff2-ab6c-4a7e-bc10-5be8c79fc09b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-723b8-predictor-66b5f9879d namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 072de111-fcd6-47e9-bda8-57c287b0bea0 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: 'Created pod: raw-sklearn-723b8-predictor-66b5f9879d-nssdq' metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:45Z" name: raw-sklearn-723b8-predictor-66b5f9879d.18d677c415ed79ba namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 965e0f21-8af7-4805-8a42-51a9ab3d1a6f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-723b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: 5eea242a-d8ce-460f-83b1-5394d6f1b019 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Scaled up replica set raw-sklearn-723b8-predictor-66b5f9879d from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:45Z" name: raw-sklearn-723b8-predictor.18d677c4128b4047 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 4fb050f7-304b-4dcc-8437-31fc1d1c7798 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-723b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 87f21305-2917-4263-a5cb-4b15478ff732 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:15Z" name: raw-sklearn-723b8-predictor.18d677c791076198 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 07502391-2428-4f91-a33f-b417764b9c36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-723b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 87f21305-2917-4263-a5cb-4b15478ff732 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:15Z" name: raw-sklearn-723b8-predictor.18d677c7918e9d67 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: af086b5b-0911-448e-9291-05225c6e165c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:40:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-723b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 87f21305-2917-4263-a5cb-4b15478ff732 kind: Event lastTimestamp: "2026-09-18T16:41:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:16Z" name: raw-sklearn-723b8-predictor.18d677ce8ef56fbe namespace: kserve-ci-e2e-test resourceVersion: "21936" uid: d07548fa-b608-4f87-8024-df3dc215cd2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:40:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-723b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 87f21305-2917-4263-a5cb-4b15478ff732 kind: Event lastTimestamp: "2026-09-18T16:41:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:16Z" name: raw-sklearn-723b8-predictor.18d677ce8f676f1f namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: f95d5a2a-257a-470c-98a3-1a15680c3b3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-723b8 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 5da20563-563b-49b9-b0c9-bb1ab198c347 kind: Event lastTimestamp: "2026-09-18T16:41:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:41:08Z" name: raw-sklearn-723b8.18d677c4128c9950 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 9d8c49f5-8716-410d-a29f-5521e67c8428 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-723b8 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 5da20563-563b-49b9-b0c9-bb1ab198c347 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: 'Failed to update status for InferenceService "raw-sklearn-723b8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-723b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:45Z" name: raw-sklearn-723b8.18d677c4130fbea8 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 7d0931ba-5173-40c8-9755-13c453f7efd0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-723b8 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 5da20563-563b-49b9-b0c9-bb1ab198c347 kind: Event lastTimestamp: "2026-09-18T16:41:08Z" message: InferenceService [raw-sklearn-723b8] is Ready metadata: creationTimestamp: "2026-09-18T16:41:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:41:08Z" name: raw-sklearn-723b8.18d677d75fa26d5d namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: ee4d7c79-f050-432b-9cef-2258c2113ffc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:41:21.705840Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 to ip-10-0-136-96.ec2.internal metadata: creationTimestamp: "2026-09-18T16:41:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:41:21Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677da5ef215bd namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: 4b762ad3-9397-418b-963a-318a2781571e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6447c659d7-c6bmn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:22Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:22Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:41:22Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677da80caf65a namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 96641fa9-0b88-44cc-a752-ee28f0e9bce2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:22Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:22Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677da8281bb21 namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 9d30dba5-f96c-4495-82cd-f6f601b1486c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:22Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677da85591e10 namespace: kserve-ci-e2e-test resourceVersion: "22043" uid: dfb217f1-2ee1-44a2-9fee-5c5fb7a29c44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:22Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677da85d0e4ca namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 583514c2-0f71-4785-8894-b2c251af1d7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:26Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:26Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677db66ab9a42 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: f726e659-4239-4632-a160-cebace28758b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:26Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677db6a5472a3 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: 649a94d7-e09c-4b72-a59f-b4187b279dff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:26Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677db6acfe02b namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 298009c0-11d4-4642-97c4-0af2871c56fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:26Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677db6ad9cfbf namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: 809cda56-10a7-4ab6-8c89-5a94bfcc0210 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:26Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677db6d8a3310 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: af39104f-bc48-47a6-bf8b-b74ebf9d2422 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:41:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:41:26Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677db6e1403b1 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: c5dbc2ae-55f9-49e5-bfc7-c4ed5561b99a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:41:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:42:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:56Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677dbde831549 namespace: kserve-ci-e2e-test resourceVersion: "22782" uid: febca90e-5869-44e7-9d78-31a45c75af61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:42:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:52Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677ef74a8dfaa namespace: kserve-ci-e2e-test resourceVersion: "22741" uid: 4c44c3ee-0220-4549-bd81-9dddb14ec65b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:42:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:52Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677ef74aa3865 namespace: kserve-ci-e2e-test resourceVersion: "22742" uid: b7ac7182-253c-4278-aaab-3da080bb5b89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 910f5941-54c7-4a6e-bb72-24c9b100edab kind: Event lastTimestamp: "2026-09-18T16:42:54Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:42:54Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527.18d677efe4c3ed86 namespace: kserve-ci-e2e-test resourceVersion: "22765" uid: ffb10609-101e-482a-b24d-f688521c02eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-96.ec2.internal source: component: kubelet host: ip-10-0-136-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-58298-predictor-97d84ccc4 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: f5a1916f-0a01-44ce-bd56-abcd9f2415f8 kind: Event lastTimestamp: "2026-09-18T16:41:21Z" message: 'Created pod: raw-sklearn-runtime-58298-predictor-97d84ccc4-pg527' metadata: creationTimestamp: "2026-09-18T16:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:21Z" name: raw-sklearn-runtime-58298-predictor-97d84ccc4.18d677da5e8784dc namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 3a86fb8f-75b2-4a30-aac0-aac9bb57351f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-58298-predictor namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: 3c6fa92d-a1e7-4d7b-90d1-76d986b5c5cb kind: Event lastTimestamp: "2026-09-18T16:41:21Z" message: Scaled up replica set raw-sklearn-runtime-58298-predictor-97d84ccc4 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:21Z" name: raw-sklearn-runtime-58298-predictor.18d677da5b29bfb2 namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 1768e47b-eb0f-4d91-9758-850d41226764 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:41:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-58298-predictor namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 1cbf1b33-abc7-4b5b-b17a-560bfc232387 kind: Event lastTimestamp: "2026-09-18T16:42:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:06Z" name: raw-sklearn-runtime-58298-predictor.18d677ddd9c7bd7a namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: 561550ee-aead-411e-aea1-1f95748fc982 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:41:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-58298-predictor namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 1cbf1b33-abc7-4b5b-b17a-560bfc232387 kind: Event lastTimestamp: "2026-09-18T16:42:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:06Z" name: raw-sklearn-runtime-58298-predictor.18d677ddda2fe661 namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 0697140d-9be9-4190-802c-863cd214c204 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:42:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-58298-predictor namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 1cbf1b33-abc7-4b5b-b17a-560bfc232387 kind: Event lastTimestamp: "2026-09-18T16:42:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:51Z" name: raw-sklearn-runtime-58298-predictor.18d677e857f84206 namespace: kserve-ci-e2e-test resourceVersion: "22717" uid: 01fe868c-a2df-49bc-97ef-e1e36c9b9389 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:42:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-58298-predictor namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 1cbf1b33-abc7-4b5b-b17a-560bfc232387 kind: Event lastTimestamp: "2026-09-18T16:42:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:42:51Z" name: raw-sklearn-runtime-58298-predictor.18d677e8586de1bf namespace: kserve-ci-e2e-test resourceVersion: "22718" uid: 39fcda72-4e3c-46d3-a112-344ab7f21c6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:41:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-58298 namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: e41443d7-8182-4741-a37f-22a0583c7403 kind: Event lastTimestamp: "2026-09-18T16:42:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:42:46Z" name: raw-sklearn-runtime-58298.18d677da5b3b13eb namespace: kserve-ci-e2e-test resourceVersion: "22681" uid: bb8df978-da26-4160-abf9-9d6756cb3fed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:41:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-58298 namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: e41443d7-8182-4741-a37f-22a0583c7403 kind: Event lastTimestamp: "2026-09-18T16:41:21Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-58298": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-58298": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:41:21Z" name: raw-sklearn-runtime-58298.18d677da5be5f93e namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 0f7f0bd1-c0dd-4d67-9956-ea740065a228 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-58298 namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: e41443d7-8182-4741-a37f-22a0583c7403 kind: Event lastTimestamp: "2026-09-18T16:42:46Z" message: InferenceService [raw-sklearn-runtime-58298] is Ready metadata: creationTimestamp: "2026-09-18T16:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:42:46Z" name: raw-sklearn-runtime-58298.18d677ee13461a67 namespace: kserve-ci-e2e-test resourceVersion: "22680" uid: 111f1143-fd7c-4bb8-a516-760130fa5c83 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "26821"