---
apiVersion: v1
items:
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:11:45.840071Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-7c7f45c769-cd7lp
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d63807477e9dd3
    namespace: kserve-ci-e2e-test
    resourceVersion: "21420"
    uid: 682428ba-6495-4580-b287-9797ffed497e
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21424"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Add eth0 [10.132.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d63807695c59be
    namespace: kserve-ci-e2e-test
    resourceVersion: "21460"
    uid: c91104a1-dd8e-4f10-8e14-87872e0277a2
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638076aca21bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21462"
    uid: 27235298-d1ff-4bcd-8c8e-500f9a1caf58
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638076db6192d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21463"
    uid: e2d3dea8-2261-475f-8a4a-801bd2ad7638
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638076e2c689b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21464"
    uid: b615c525-1734-4854-ac17-d44eac3eb02f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d6380868cf9787
    namespace: kserve-ci-e2e-test
    resourceVersion: "21512"
    uid: 6583123c-c19b-4101-9ea5-3f109106ca5f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638086d044f48
    namespace: kserve-ci-e2e-test
    resourceVersion: "21516"
    uid: b44f42ea-2385-48b3-9659-d0b2af69fb6f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638086d7a91fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21519"
    uid: 63a5051e-9466-46f2-911f-e99af7369b2c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638086d88df27
    namespace: kserve-ci-e2e-test
    resourceVersion: "21520"
    uid: b12ed3bf-19a7-4de8-96d3-062890bc266f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d6380870e6733e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21526"
    uid: 68a2d8dd-1b9f-4a87-88df-7663e8c00aa0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d638087175d2a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21528"
    uid: c5f3f3b1-9714-4158-b410-2db704c7a072
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:12:58Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:11:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:12:58Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d63808e0e3303d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22033"
    uid: 5ea6098f-9ed3-4eb4-b063-56ff74882e6a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:13:18Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:13:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:18Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d6381cef39125b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22209"
    uid: 0011c838-2fab-4bfe-86a0-e618cf5e159a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:13:18Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:13:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:18Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d6381cef3a54c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "22212"
    uid: c5376ec4-8ed5-4abf-97ec-3bd2a0506f83
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 2f2ddaad-0579-48f5-a8d8-ced1f2a8976e
  kind: Event
  lastTimestamp: "2026-09-17T21:13:23Z"
  message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp
    10.132.0.33:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:13:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:23Z"
    name: isvc-canary-create-predictor-7c7f45c769-cd7lp.18d6381e111e994c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22279"
    uid: 9f92803d-09ff-4982-b7e3-08e05bb764ce
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:11:45.715163Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21393"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-7c7f45c769-tq6h7
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638073ff30837
    namespace: kserve-ci-e2e-test
    resourceVersion: "21404"
    uid: 8c35c6d6-0c4e-4cac-a57f-f781310b9930
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21411"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Add eth0 [10.133.0.38/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d63807620c3f74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21449"
    uid: a2afc7b4-6f04-44d5-8d1d-3d6236bf2150
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d6380763880891
    namespace: kserve-ci-e2e-test
    resourceVersion: "21454"
    uid: 492a0286-2ed9-4273-8fbe-8eb6d009a845
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d6380765e86627
    namespace: kserve-ci-e2e-test
    resourceVersion: "21455"
    uid: cf396e24-384f-4b87-812c-3c46f9e0aff1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638076657e6f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21456"
    uid: d5633446-db78-4afc-b5e8-bedba0c88532
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d6380878a1ddd7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21532"
    uid: 38f5c28e-eca6-4b89-bc80-297befe77374
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:51Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:11:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:51Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638087bfc3832
    namespace: kserve-ci-e2e-test
    resourceVersion: "21536"
    uid: 7aee9469-218b-4eb8-8b83-45a1d6e0b1c1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:51Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:11:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:51Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638087c6aef12
    namespace: kserve-ci-e2e-test
    resourceVersion: "21537"
    uid: 3afaea2a-6943-4870-a1bc-e445c66fcc0e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:51Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:51Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638087c746d39
    namespace: kserve-ci-e2e-test
    resourceVersion: "21539"
    uid: f434c805-2648-49ca-9e23-f6b9f5a17918
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:51Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:11:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:51Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638087f397e69
    namespace: kserve-ci-e2e-test
    resourceVersion: "21541"
    uid: 65de11e0-b46b-4b95-8532-bb745884b8e8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:11:51Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:11:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:51Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d638087fa84441
    namespace: kserve-ci-e2e-test
    resourceVersion: "21542"
    uid: d59b98e6-d8da-4ba1-9db6-85c87fada20e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:12:58Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:11:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:12:58Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d63808f07d24cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22038"
    uid: a073ea36-07e6-4fac-b64a-b4b73dea9160
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:13:20Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:13:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:20Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d6381d681a6bba
    namespace: kserve-ci-e2e-test
    resourceVersion: "22256"
    uid: 4089d598-3cfb-48d7-837b-e373cb8ac77e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:13:20Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:13:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:20Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d6381d681b0177
    namespace: kserve-ci-e2e-test
    resourceVersion: "22258"
    uid: a4df4c28-a0b2-4a10-8ae7-94d9022c6c79
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 1b7d2fc8-6813-41ec-9b52-bae5676260de
  kind: Event
  lastTimestamp: "2026-09-17T21:13:23Z"
  message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp
    10.133.0.38:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:13:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:23Z"
    name: isvc-canary-create-predictor-7c7f45c769-tq6h7.18d6381e20ad281a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22283"
    uid: 9d1978c1-9c25-470a-bc23-03edb889de03
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-7c7f45c769
    namespace: kserve-ci-e2e-test
    resourceVersion: "21385"
    uid: bd1ee11c-eee3-4393-bf21-2a2d8b5a3c37
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'Created pod: isvc-canary-create-predictor-7c7f45c769-tq6h7'
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-predictor-7c7f45c769.18d638073f3731da
    namespace: kserve-ci-e2e-test
    resourceVersion: "21399"
    uid: 223bbde7-cba3-4602-9288-a43d44aface8
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-7c7f45c769
    namespace: kserve-ci-e2e-test
    resourceVersion: "21385"
    uid: bd1ee11c-eee3-4393-bf21-2a2d8b5a3c37
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'Created pod: isvc-canary-create-predictor-7c7f45c769-cd7lp'
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-predictor-7c7f45c769.18d6380746260821
    namespace: kserve-ci-e2e-test
    resourceVersion: "21419"
    uid: c280b0b6-38c2-4071-b8f4-15f50c8d9855
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:18Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-7c7f45c769
    namespace: kserve-ci-e2e-test
    resourceVersion: "22201"
    uid: bd1ee11c-eee3-4393-bf21-2a2d8b5a3c37
  kind: Event
  lastTimestamp: "2026-09-17T21:13:18Z"
  message: 'Deleted pod: isvc-canary-create-predictor-7c7f45c769-cd7lp'
  metadata:
    creationTimestamp: "2026-09-17T21:13:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:18Z"
    name: isvc-canary-create-predictor-7c7f45c769.18d6381cef09b302
    namespace: kserve-ci-e2e-test
    resourceVersion: "22206"
    uid: fa9e7caa-ffe7-4e38-9fd3-a4e6bd88c187
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21384"
    uid: 0f68d867-d25c-4d8a-a731-a8f7c0d428af
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: Scaled up replica set isvc-canary-create-predictor-7c7f45c769 from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-predictor.18d638073aea804e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21387"
    uid: 27fe1520-543a-4621-bc96-3804df369af0
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:18Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22197"
    uid: 0f68d867-d25c-4d8a-a731-a8f7c0d428af
  kind: Event
  lastTimestamp: "2026-09-17T21:13:18Z"
  message: Scaled down replica set isvc-canary-create-predictor-7c7f45c769 from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:13:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:18Z"
    name: isvc-canary-create-predictor.18d6381cee6371a8
    namespace: kserve-ci-e2e-test
    resourceVersion: "22202"
    uid: 2ce5227f-c618-4f62-95f9-05cab0725164
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:11:45.715541Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21392"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d638073ff8c74f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21403"
    uid: d51ca062-7b1a-4a02-a944-19735750d5b6
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21413"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Add eth0 [10.132.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d638076210d658
    namespace: kserve-ci-e2e-test
    resourceVersion: "21450"
    uid: e08a2a47-f4eb-43e2-af9d-6142266bebe6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d63807638c96bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21453"
    uid: 4570d28a-e44f-4fa6-ad4d-9468a328171b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6380766d2e55a
    namespace: kserve-ci-e2e-test
    resourceVersion: "21457"
    uid: 56c80428-5b1a-4a65-9921-3aa3e44b037a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6380767488fa8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21458"
    uid: 517517cc-9c23-49e4-a047-54536e15bf9e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6380868e5c5f1
    namespace: kserve-ci-e2e-test
    resourceVersion: "21513"
    uid: 4fb8d9dc-7fca-4df5-9443-2f2c4eb53f79
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d638086d0e9341
    namespace: kserve-ci-e2e-test
    resourceVersion: "21518"
    uid: 2da26dea-60e5-4fb4-b2e5-0d4bb1a2ef6c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d638086d88f72e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21521"
    uid: dde9ea9e-2e3b-4d1b-9737-ba70db4f8efd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d638086d936ccd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21522"
    uid: 033242e7-713e-4161-8bbd-8445106c4708
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6380870daf4c1
    namespace: kserve-ci-e2e-test
    resourceVersion: "21525"
    uid: 5d1d1e9b-5db4-4126-8b36-59f15139e668
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:11:50Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:11:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:50Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d638087166750b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21527"
    uid: 00c0485e-2381-4894-b3ad-e3a1cb550110
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:13:08Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:11:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:08Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d63808e0e3f521
    namespace: kserve-ci-e2e-test
    resourceVersion: "22105"
    uid: 82742b81-cbe1-4e93-a9a3-776801f37cfd
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:13:20Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:13:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:20Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6381d67614655
    namespace: kserve-ci-e2e-test
    resourceVersion: "22253"
    uid: 71ed2fb1-0813-4231-bfa3-3810759ccc12
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:13:20Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:13:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:20Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6381d6761c86c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22255"
    uid: 9de0fd49-005a-4e1e-8ac9-e35dee6cefc1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21398"
    uid: e72d4ce2-ee89-4167-9628-05705846e4ca
  kind: Event
  lastTimestamp: "2026-09-17T21:13:23Z"
  message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp
    10.132.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:13:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:23Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf.18d6381e111e0dd2
    namespace: kserve-ci-e2e-test
    resourceVersion: "22278"
    uid: f4ca8b78-e2e3-419d-8f1c-7f3fdc979fca
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-v2-predictor-bfcdb48dd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21376"
    uid: 7494a8b9-b962-4bac-97ba-8c2701668bba
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'Created pod: isvc-canary-create-v2-predictor-bfcdb48dd-jggsf'
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-v2-predictor-bfcdb48dd.18d638073ed9370b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21394"
    uid: 75d3e53c-f8d5-41c9-808f-bc988138c765
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21374"
    uid: 01526c9d-d1ed-4134-8d5b-66afe881504c
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: Scaled up replica set isvc-canary-create-v2-predictor-bfcdb48dd from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create-v2-predictor.18d63807387c70c7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21377"
    uid: b44d0b2e-71ff-445c-ba7b-13dbad55c099
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21366"
    uid: a7398f49-b27c-4995-a424-27cbaa8c90dd
  kind: Event
  lastTimestamp: "2026-09-17T21:13:19Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:13:19Z"
    name: isvc-canary-create.18d638073a788f09
    namespace: kserve-ci-e2e-test
    resourceVersion: "22218"
    uid: 02ce1f9e-b8bd-4079-a7cd-ae9ce844d039
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21366"
    uid: a7398f49-b27c-4995-a424-27cbaa8c90dd
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'Failed to update status for InferenceService "isvc-canary-create": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create.18d638073af1d05c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21426"
    uid: af426d4a-728e-4634-9bbe-7c5cb54b8b03
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:45Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21366"
    uid: a7398f49-b27c-4995-a424-27cbaa8c90dd
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:11:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: isvc-canary-create.18d638073af21428
    namespace: kserve-ci-e2e-test
    resourceVersion: "21427"
    uid: 755b82db-ce13-42d1-a320-96444b8533a1
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:09Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "22123"
    uid: a7398f49-b27c-4995-a424-27cbaa8c90dd
  kind: Event
  lastTimestamp: "2026-09-17T21:13:09Z"
  message: InferenceService [isvc-canary-create] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:13:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:13:09Z"
    name: isvc-canary-create.18d6381aa82b6181
    namespace: kserve-ci-e2e-test
    resourceVersion: "22124"
    uid: 55ed62ca-67ad-4131-ae57-d62c517a1ce8
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:13:25.944477Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22333"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381e9614f058
    namespace: kserve-ci-e2e-test
    resourceVersion: "22341"
    uid: 7dba8069-ce98-44ec-919e-5e155a26099e
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381e9d182a42
    namespace: kserve-ci-e2e-test
    resourceVersion: "22359"
    uid: 2472e4ad-3022-40e0-b2bf-05fb708326ca
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:27Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22350"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:27Z"
  message: Add eth0 [10.132.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:13:27Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381edbd5157a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22388"
    uid: 8964416a-16ea-40b6-895f-0502db22c0df
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:27Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:27Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381edd530fec
    namespace: kserve-ci-e2e-test
    resourceVersion: "22390"
    uid: a9603383-8dd8-4733-b4a5-2ae36a969e34
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:27Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:27Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381ee0298125
    namespace: kserve-ci-e2e-test
    resourceVersion: "22392"
    uid: 4a4b3ac0-cf89-4e9c-b0e5-2dac76feafff
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:27Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:27Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381ee09a6695
    namespace: kserve-ci-e2e-test
    resourceVersion: "22393"
    uid: d68d8715-4726-486f-9780-ba3606ce9e13
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381fcbf8717f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22433"
    uid: 99870571-0b1f-4c78-b594-a6d4d2af8b9a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381fcf533e6a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22436"
    uid: 08e99a2c-5ede-4e34-8ea6-d61d1a535c7d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381fcfc51ae5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22437"
    uid: da5412f4-a391-45d8-8322-2fde241af84b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381fcfd13223
    namespace: kserve-ci-e2e-test
    resourceVersion: "22438"
    uid: fbc427a7-32b2-4c02-bb99-25ae2d68a6ed
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381fd2a15356
    namespace: kserve-ci-e2e-test
    resourceVersion: "22440"
    uid: b071df4e-6e75-498c-9f5e-b0af52d0c94e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6381fd31125f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22441"
    uid: 126a0df0-7157-4403-a9e2-4456632d7ce0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:14:41Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:13:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:41Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6382043d2e319
    namespace: kserve-ci-e2e-test
    resourceVersion: "22961"
    uid: c20920e1-dd5d-4fc2-b244-a9ed5ab98ffb
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6383397f819d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "23119"
    uid: 238105f2-44c2-4991-99d7-fcdb1c7906c4
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d6383397f88599
    namespace: kserve-ci-e2e-test
    resourceVersion: "23121"
    uid: dc607422-ad03-4481-921f-58fd911a885d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:14:59Z"
  message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp
    10.132.0.34:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:14:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:59Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d638344a09a28b
    namespace: kserve-ci-e2e-test
    resourceVersion: "23164"
    uid: dbfade71-14cd-4f7b-9f26-a60683a6b1a0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: ffb8954a-ca80-459c-974e-0d549edd1e6c
  kind: Event
  lastTimestamp: "2026-09-17T21:15:02Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: i/o timeout'
  metadata:
    creationTimestamp: "2026-09-17T21:15:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:02Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-8pzzt.18d63834fd4e0345
    namespace: kserve-ci-e2e-test
    resourceVersion: "23202"
    uid: a9694032-eb4a-4c05-a4a1-f83e25b41841
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:13:26.044061Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22352"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381e9c049213
    namespace: kserve-ci-e2e-test
    resourceVersion: "22356"
    uid: b102371f-e21b-4eb5-92e6-bbc414642513
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22364"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: Add eth0 [10.133.0.40/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381ebdd1460c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22380"
    uid: 220fceaa-533d-4820-8b1c-055b4a705779
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381ebf1fd39b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22382"
    uid: affa30d8-c89f-4a10-b200-b30e62327ddd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381ec197c7b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22383"
    uid: 87e4ccda-f376-43c6-b916-1d687bf96081
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381ec209e2dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "22384"
    uid: b0a00a13-0a8f-4257-8aea-eac3473f9814
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381fd8412ebb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22442"
    uid: bb1d2c11-3dcc-44cb-acb8-3638a8afe4e2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381fdc33cddd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22448"
    uid: dd4151a9-5e4f-421b-9395-3ee663a26fa0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381fdcaaccbd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22453"
    uid: 68ce1921-02b1-464a-bf35-45c9b68b1ec2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381fdcb5f658
    namespace: kserve-ci-e2e-test
    resourceVersion: "22454"
    uid: 8ec131be-3f2d-4a63-92f8-73e7c283a459
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381fdfe2da2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22456"
    uid: 0377c4ad-259b-4e1a-a56a-3f2d553e84f2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d6381fe057722a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22458"
    uid: aed051b0-b183-455e-882e-1536b1cba23a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:14:42Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:13:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:42Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d63820147aa4bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22974"
    uid: 3ea3ccc6-4c7d-48a6-be0a-7466260cf3e0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:14:49Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:14:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:49Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d638320b507b79
    namespace: kserve-ci-e2e-test
    resourceVersion: "23033"
    uid: 39455538-957e-4a2a-9145-b41709a27bff
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:14:49Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:14:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:49Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d638320b50c33a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23035"
    uid: 9c6c451c-51e5-482d-b2e5-67af59650e63
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22354"
    uid: 7f57872e-bcb1-4648-895a-320034d57ac4
  kind: Event
  lastTimestamp: "2026-09-17T21:14:53Z"
  message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp
    10.133.0.40:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:14:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:53Z"
    name: isvc-canary-promote-predictor-78b4c78fb9-m4vvd.18d63832f0ab4f95
    namespace: kserve-ci-e2e-test
    resourceVersion: "23081"
    uid: 4dd13fd5-2179-4c39-ae74-749273d9bd1d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-78b4c78fb9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22325"
    uid: 4b50e6ef-b9e3-439d-a694-d1710fb61023
  kind: Event
  lastTimestamp: "2026-09-17T21:13:25Z"
  message: 'Created pod: isvc-canary-promote-predictor-78b4c78fb9-8pzzt'
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote-predictor-78b4c78fb9.18d6381e95389b79
    namespace: kserve-ci-e2e-test
    resourceVersion: "22338"
    uid: 13d6d89a-d9a5-4e2a-b445-0d5021807e35
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-78b4c78fb9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22325"
    uid: 4b50e6ef-b9e3-439d-a694-d1710fb61023
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: 'Created pod: isvc-canary-promote-predictor-78b4c78fb9-m4vvd'
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-predictor-78b4c78fb9.18d6381e9b59a69c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22353"
    uid: 9926b512-ad4f-4cf7-aabb-e4f648eb1b86
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-78b4c78fb9
    namespace: kserve-ci-e2e-test
    resourceVersion: "23026"
    uid: 4b50e6ef-b9e3-439d-a694-d1710fb61023
  kind: Event
  lastTimestamp: "2026-09-17T21:14:49Z"
  message: 'Deleted pod: isvc-canary-promote-predictor-78b4c78fb9-m4vvd'
  metadata:
    creationTimestamp: "2026-09-17T21:14:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:14:49Z"
    name: isvc-canary-promote-predictor-78b4c78fb9.18d638320b20f23c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23029"
    uid: 8038829e-1c41-49f8-a495-81326c5dc63a
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22323"
    uid: 9f302b46-7995-4b1b-8880-f9bc032c5801
  kind: Event
  lastTimestamp: "2026-09-17T21:13:25Z"
  message: Scaled up replica set isvc-canary-promote-predictor-78b4c78fb9 from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote-predictor.18d6381e91ac9c06
    namespace: kserve-ci-e2e-test
    resourceVersion: "22327"
    uid: 1ecf9b22-0335-4ae3-9d50-25322a5ca5a1
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23022"
    uid: 9f302b46-7995-4b1b-8880-f9bc032c5801
  kind: Event
  lastTimestamp: "2026-09-17T21:14:49Z"
  message: Scaled down replica set isvc-canary-promote-predictor-78b4c78fb9 from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:14:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:14:49Z"
    name: isvc-canary-promote-predictor.18d638320a9a7e5d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23028"
    uid: 36e54606-75f2-4c27-ae63-787f4fece72b
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:13:25.944285Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22332"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381e96121045
    namespace: kserve-ci-e2e-test
    resourceVersion: "22340"
    uid: afefdb7b-20b3-4bbe-bc17-766bdfdfa02d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22351"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: Add eth0 [10.133.0.39/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381eb7ca6825
    namespace: kserve-ci-e2e-test
    resourceVersion: "22375"
    uid: ab38a518-8ee4-4485-89f0-151675cde894
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381eb924083f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22377"
    uid: b821a98b-2cf3-41b0-a3a5-035cc0ba096d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381ebc28b498
    namespace: kserve-ci-e2e-test
    resourceVersion: "22378"
    uid: a75c1d9e-f9a0-4a17-8e6c-ed018207ded3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:26Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:26Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381ebc9c1aef
    namespace: kserve-ci-e2e-test
    resourceVersion: "22379"
    uid: cfbf98fd-9a04-4e84-b75d-febf669ca1d2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381fd856dc97
    namespace: kserve-ci-e2e-test
    resourceVersion: "22443"
    uid: 2b03d7f6-2799-4cf3-8964-2804028a9603
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381fdc2a1567
    namespace: kserve-ci-e2e-test
    resourceVersion: "22447"
    uid: a796582a-f868-4009-aab6-aaa4630ad025
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381fdc9dcfa3
    namespace: kserve-ci-e2e-test
    resourceVersion: "22449"
    uid: 74f06491-14d5-4cf9-be4b-87f27faedffa
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381fdca7f60e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22450"
    uid: 15714a34-932b-4311-ae5c-c33e0bcd58e3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381fdfd5a209
    namespace: kserve-ci-e2e-test
    resourceVersion: "22455"
    uid: 1b01675b-daba-4124-b89a-ccd802901b5e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:31Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:13:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:13:31Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6381fe049b17b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22457"
    uid: 35a1a528-0c04-424c-a072-46f3ddfa7eba
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:16:39Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:13:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:39Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6382050511b54
    namespace: kserve-ci-e2e-test
    resourceVersion: "23897"
    uid: 7219c55a-cf02-4889-aea6-27494bb7e3b3
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:16:36Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:16:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:36Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6384aed65d828
    namespace: kserve-ci-e2e-test
    resourceVersion: "23875"
    uid: a564c61a-13d1-4633-91a0-d0dce8ebd7cc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:16:36Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:16:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:36Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6384aed666356
    namespace: kserve-ci-e2e-test
    resourceVersion: "23877"
    uid: 6d0089a6-cb84-4c36-a820-35b9b93f5e5b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:39Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s
    namespace: kserve-ci-e2e-test
    resourceVersion: "22335"
    uid: 762d65aa-f29f-4025-a118-c61ceeac20b9
  kind: Event
  lastTimestamp: "2026-09-17T21:16:39Z"
  message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp
    10.133.0.39:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:16:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:39Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s.18d6384b9ef3bc41
    namespace: kserve-ci-e2e-test
    resourceVersion: "23896"
    uid: 3f2b9852-605a-430c-9e84-3c0d3edf2ac5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:14:56.267302Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23124"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d638339dbdee07
    namespace: kserve-ci-e2e-test
    resourceVersion: "23129"
    uid: 58d074b8-d7b9-42bb-a8bf-ae67eb6a3169
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23133"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Add eth0 [10.132.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63833bfb24e47
    namespace: kserve-ci-e2e-test
    resourceVersion: "23141"
    uid: 511df4dc-e916-4d26-a425-b6488a9016e4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63833c180f13d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23143"
    uid: 1690d264-5029-43db-999f-f0e959353a5a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63833c3fee37d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23144"
    uid: aa167016-0277-4976-b7a3-b747a49ea355
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63833c4707585
    namespace: kserve-ci-e2e-test
    resourceVersion: "23145"
    uid: 69963724-b56d-436d-98ff-e6b664fbdf33
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:15:01Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:15:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:01Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63834d7015ff0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23188"
    uid: 30f95887-60e5-4664-8a63-ba775355183a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:15:01Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:15:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:01Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63834d9dcaf30
    namespace: kserve-ci-e2e-test
    resourceVersion: "23191"
    uid: 25b18ad3-d337-452c-b37d-6bdbf92189c9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:15:01Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:15:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:01Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63834da4ff3d5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23194"
    uid: 316ed1f7-cdbe-4969-b8d1-46de35c7134f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:15:01Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:15:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:01Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63834da5b5aea
    namespace: kserve-ci-e2e-test
    resourceVersion: "23195"
    uid: 5f59e77e-08be-4454-b491-ac82ee0270d0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:15:01Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:15:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:01Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63834dcc1354e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23196"
    uid: 7c327e64-1803-4545-aaf2-30b9ddf040cf
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:15:01Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:15:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:15:01Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d63834dd3182c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "23198"
    uid: 2ca5095e-6c82-403c-9461-b71b062bb219
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:15:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:16:39Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:15:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:39Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d638354ee760d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "23894"
    uid: 0065e58d-9455-44bf-8f9a-29e29831cf9a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:16:36Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:16:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:36Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d6384aecff14dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "23873"
    uid: 996540c2-e577-4791-a615-0d537ee4af20
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:16:36Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:16:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:36Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d6384aecfffcef
    namespace: kserve-ci-e2e-test
    resourceVersion: "23874"
    uid: f6775c65-8fb9-47bc-91a3-5dfb706bdb76
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:39Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "23126"
    uid: 40aa0f85-2c9d-4ac2-9eb2-d42e9f24ae93
  kind: Event
  lastTimestamp: "2026-09-17T21:16:39Z"
  message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp
    10.132.0.35:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:16:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:39Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w.18d6384ba9326d73
    namespace: kserve-ci-e2e-test
    resourceVersion: "23898"
    uid: ad0e7305-ec21-4b77-ba9c-8f72f8414fd9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22316"
    uid: 42aa62d8-5301-4134-8ce8-3b7968042be9
  kind: Event
  lastTimestamp: "2026-09-17T21:13:25Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-79d9bd5fdb-6nx2s'
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb.18d6381e95140b91
    namespace: kserve-ci-e2e-test
    resourceVersion: "22334"
    uid: e220cd7b-16ca-4c6c-8d7c-5069df2df1c5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb
    namespace: kserve-ci-e2e-test
    resourceVersion: "23109"
    uid: 42aa62d8-5301-4134-8ce8-3b7968042be9
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-79d9bd5fdb-wbc5w'
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor-79d9bd5fdb.18d638339d08973c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23128"
    uid: 5791c5e8-a1d3-4605-ab26-3d07eae4111b
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22314"
    uid: eda97afd-47de-46fa-9985-1cd3e6ce4dcd
  kind: Event
  lastTimestamp: "2026-09-17T21:13:25Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-79d9bd5fdb from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote-v2-predictor.18d6381e8fd6cb64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22317"
    uid: 8f809719-205d-47fb-baf8-e864e1bfde1f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23104"
    uid: eda97afd-47de-46fa-9985-1cd3e6ce4dcd
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-79d9bd5fdb from
    1 to 2
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote-v2-predictor.18d6383395da9da5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23111"
    uid: f74f0597-b8f5-4014-a2ff-2a3ea32257e0
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 22
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22307"
    uid: dc93ad1b-1306-4380-b8cf-0ad4921f49ad
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote.18d6381e9168b786
    namespace: kserve-ci-e2e-test
    resourceVersion: "23135"
    uid: b7a05e27-f349-4353-9528-0aafe66e1773
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22307"
    uid: dc93ad1b-1306-4380-b8cf-0ad4921f49ad
  kind: Event
  lastTimestamp: "2026-09-17T21:13:25Z"
  message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote.18d6381e91c70735
    namespace: kserve-ci-e2e-test
    resourceVersion: "22328"
    uid: 7ea11e13-45ce-4cff-a1a5-d0fd19431a62
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:13:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22307"
    uid: dc93ad1b-1306-4380-b8cf-0ad4921f49ad
  kind: Event
  lastTimestamp: "2026-09-17T21:13:25Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:13:25Z"
    name: isvc-canary-promote.18d6381e91c73f44
    namespace: kserve-ci-e2e-test
    resourceVersion: "22330"
    uid: 6886763e-24f1-4349-a3c6-f310176d6d44
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "23066"
    uid: dc93ad1b-1306-4380-b8cf-0ad4921f49ad
  kind: Event
  lastTimestamp: "2026-09-17T21:16:29Z"
  message: InferenceService [isvc-canary-promote] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:14:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:16:29Z"
    name: isvc-canary-promote.18d63832754200a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "23808"
    uid: 96cb96e4-af0d-42bc-8123-897d4b6aaa69
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:14:56Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "23123"
    uid: dc93ad1b-1306-4380-b8cf-0ad4921f49ad
  kind: Event
  lastTimestamp: "2026-09-17T21:14:56Z"
  message: 'InferenceService [isvc-canary-promote] is no longer Ready because of:
    PredictorReady, Ready, Stopped'
  metadata:
    creationTimestamp: "2026-09-17T21:14:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:14:56Z"
    name: isvc-canary-promote.18d638339cc64e69
    namespace: kserve-ci-e2e-test
    resourceVersion: "23125"
    uid: 1cd45e7b-8fa3-4100-a511-c5274f6ddf68
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:01:07.450283Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14931"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-97977b6b5-4dlsq
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63772a465e3ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "14934"
    uid: 1d2afc14-9f57-4172-a4b3-b9ddcc605c46
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14943"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Add eth0 [10.133.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63772c633c3e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14992"
    uid: 41a43066-6df3-486e-8839-fe4328da0e4f
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63772c795b2a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "14995"
    uid: d1406108-4ee8-4e40-9635-fd674fc70cc7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63772cb3849a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "14997"
    uid: 3be533f6-01b3-4cd4-ae67-011e5b4a8f4f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63772cbac710f
    namespace: kserve-ci-e2e-test
    resourceVersion: "14998"
    uid: c4f6796d-508f-495f-b703-90b2d0a22494
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:12Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
  metadata:
    creationTimestamp: "2026-09-17T21:01:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:12Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63773bc1541d5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15050"
    uid: 8eb930ae-501b-4ec1-9e5b-569010c946cd
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    in 19.101s (19.101s including waiting). Image size: 1604543374 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d637782e9cb826
    namespace: kserve-ci-e2e-test
    resourceVersion: "15208"
    uid: 1027aa60-5bb8-45f8-8157-32c2a236d612
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d6377831d6825e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15215"
    uid: 59a09bf8-f633-4a2f-a0af-e08ec08053e5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d6377832462faa
    namespace: kserve-ci-e2e-test
    resourceVersion: "15216"
    uid: db9ad1c1-624b-4694-af0a-ea6ed78c5609
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d637783251bd45
    namespace: kserve-ci-e2e-test
    resourceVersion: "15217"
    uid: c14bca49-b06e-4ce9-8436-49b74ec2e4bf
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.816s (2.816s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63778da326469
    namespace: kserve-ci-e2e-test
    resourceVersion: "15243"
    uid: 709746b2-0b5e-443e-9381-16430b78b9fc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63778dced982d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15244"
    uid: f6fb9911-c9ce-42a3-8c84-67b328899eac
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d63778dd5f4397
    namespace: kserve-ci-e2e-test
    resourceVersion: "15246"
    uid: bacafc15-6f2b-4a22-ad40-1075afe758e9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:02:41Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:02:41Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d6377922c265d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15776"
    uid: 7a29f4e2-035b-41ea-8803-16351707fb73
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:03:00Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:03:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:00Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d6378cf5940dce
    namespace: kserve-ci-e2e-test
    resourceVersion: "15952"
    uid: e1d373d2-24da-4703-8862-b40155c50d98
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:03:00Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:03:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:00Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d6378cf5946cc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15953"
    uid: 2997bdf7-7ad1-4062-b678-86cd3c75f78f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 3f2467cf-97ac-40f4-8c8c-3468b677908c
  kind: Event
  lastTimestamp: "2026-09-17T21:03:01Z"
  message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp
    10.133.0.26:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:03:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:01Z"
    name: isvc-canary-rollback-predictor-97977b6b5-4dlsq.18d6378d2902b8ce
    namespace: kserve-ci-e2e-test
    resourceVersion: "15970"
    uid: d2c48a09-248c-46d8-9610-34a0ab4262e6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:01:07.559376Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14953"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-97977b6b5-cp7gp
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63772aae67620
    namespace: kserve-ci-e2e-test
    resourceVersion: "14959"
    uid: 8562bce5-7c37-4740-b26b-c3b6b7d88228
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14974"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Add eth0 [10.132.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63772ccf47707
    namespace: kserve-ci-e2e-test
    resourceVersion: "14999"
    uid: 533dcc27-9685-45b9-8ddf-9707ae768286
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63772ced1e7f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15007"
    uid: 91d830d3-79a2-4c03-a5da-51d197b9d38f
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:12Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    in 4.42s (4.42s including waiting). Image size: 306739624 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:12Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63773d6523294
    namespace: kserve-ci-e2e-test
    resourceVersion: "15063"
    uid: 465ef30b-4545-41da-aef4-3b38eb9710a1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:12Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:01:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:12Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63773d951a56c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15064"
    uid: 99231671-9d2b-4439-9681-2ff2cd79aed0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:12Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:01:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:12Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63773d9c00a1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15065"
    uid: c4e74576-e6d9-41aa-a3fd-c9b4279a1553
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:16Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
  metadata:
    creationTimestamp: "2026-09-17T21:01:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:16Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63774dc8001aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "15100"
    uid: f8f62650-4704-4526-be1e-9406a9ad2b16
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:30Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    in 13.767s (13.768s including waiting). Image size: 1604543374 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:30Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d6377811237cac
    namespace: kserve-ci-e2e-test
    resourceVersion: "15197"
    uid: e85affc3-4113-4048-812e-84f06d767b23
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:30Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:01:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:30Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63778144fd2b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15199"
    uid: 0eaa6592-4d80-4af6-b277-7e45bdda47ed
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:30Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:01:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:30Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d6377814c3abe6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15200"
    uid: a16f5ffc-0fb6-42e4-a652-64f730633167
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:30Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T21:01:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:30Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d6377814d096d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15201"
    uid: cb234796-ba13-4afa-b740-4b7be31b9b6a
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:33Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.194s (2.194s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:33Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d63778979cb290
    namespace: kserve-ci-e2e-test
    resourceVersion: "15230"
    uid: 07ae961d-24a0-4221-9fce-8f2f2ee26a7e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:33Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:01:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:33Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d637789b0ee3ed
    namespace: kserve-ci-e2e-test
    resourceVersion: "15231"
    uid: eb6b7a79-bc73-41f4-85de-3fdf4c3c43ec
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:01:33Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:01:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:33Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d637789b859a00
    namespace: kserve-ci-e2e-test
    resourceVersion: "15232"
    uid: ba77491f-e419-433f-b71c-2970f02dfac9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:02:41Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:02:41Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d637791346f467
    namespace: kserve-ci-e2e-test
    resourceVersion: "15773"
    uid: 26a99cee-a9a8-4a15-9483-fbeabd843350
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:04:22Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:04:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:22Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d637a00377ebc3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16796"
    uid: 3876c504-5099-4c03-9b92-79849bb8f39b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:04:22Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:04:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:22Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d637a003797747
    namespace: kserve-ci-e2e-test
    resourceVersion: "16797"
    uid: a8089600-b9e4-401f-9439-876f7b440017
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp
    namespace: kserve-ci-e2e-test
    resourceVersion: "14955"
    uid: 97bc6251-5ff3-4395-bf0e-606fdb336f51
  kind: Event
  lastTimestamp: "2026-09-17T21:04:26Z"
  message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp
    10.132.0.25:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:04:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:26Z"
    name: isvc-canary-rollback-predictor-97977b6b5-cp7gp.18d637a0e3e5024b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16829"
    uid: c4b78fb7-cf94-435c-a7ca-d690517b8725
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:03:02.270045Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15997"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-97977b6b5-lfppv
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378d60307065
    namespace: kserve-ci-e2e-test
    resourceVersion: "16001"
    uid: 41c53aae-bb43-4955-ba22-fb268c3f4092
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "16008"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Add eth0 [10.133.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378d81fd922a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16014"
    uid: a19f35ff-2fd3-4a49-9dd3-c012977276b4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378d83cd969d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16017"
    uid: b1b7df77-5be8-4a69-9732-b53f0f8451dd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378d86b595d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16019"
    uid: ba898f95-411c-41b1-8e30-f904da4f3202
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378d87251c81
    namespace: kserve-ci-e2e-test
    resourceVersion: "16020"
    uid: 9168050c-1db6-4062-9270-49833e183a0b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378ea6038c7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16063"
    uid: fcf55ccf-b97b-407b-b74b-4f9a306205f0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:03:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378ea9082e67
    namespace: kserve-ci-e2e-test
    resourceVersion: "16066"
    uid: 55e0893e-da7d-4ac4-bdeb-51395b821d46
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:03:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378ea97c5d2b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16067"
    uid: cd2126a4-dc54-4d48-92f0-8a5e3906ceb2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378ea986dd06
    namespace: kserve-ci-e2e-test
    resourceVersion: "16069"
    uid: 21e44f71-c9e7-4737-ab26-b5d08bd4dcfd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:03:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378eacb97e77
    namespace: kserve-ci-e2e-test
    resourceVersion: "16070"
    uid: 4cb9a827-e63b-4b7e-8a76-c142a40e75f6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:03:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378ead28028b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16071"
    uid: b54d4009-eb4f-47e4-9d8d-6af150aa96ca
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:04:15Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:03:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:15Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d6378f1dea36c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16731"
    uid: 5c4c8002-8d1d-46ab-916a-ce51c9d45ed2
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:04:23Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:04:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:23Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d637a02ed5ad20
    namespace: kserve-ci-e2e-test
    resourceVersion: "16803"
    uid: 12f2f75c-ebf2-43a6-bb89-e36f5ed2aacd
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:04:23Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:04:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:23Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d637a02ed70ede
    namespace: kserve-ci-e2e-test
    resourceVersion: "16804"
    uid: 97963795-bc25-4e55-a8d6-941f58c79dcc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15999"
    uid: 8ffdcb55-f68a-4cbf-941a-0a6c817afc12
  kind: Event
  lastTimestamp: "2026-09-17T21:04:25Z"
  message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp
    10.133.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:04:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:25Z"
    name: isvc-canary-rollback-predictor-97977b6b5-lfppv.18d637a0d009ec48
    namespace: kserve-ci-e2e-test
    resourceVersion: "16825"
    uid: cf2e5559-43c7-4b1b-8d1a-fef23a59b31d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-97977b6b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14913"
    uid: bb9a4eaf-f5fb-4225-bf9b-f1fa26ab97b2
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: 'Created pod: isvc-canary-rollback-predictor-97977b6b5-4dlsq'
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5.18d63772a3e616c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "14933"
    uid: 94539d27-8b8a-4f59-bc07-c98804469720
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-97977b6b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14913"
    uid: bb9a4eaf-f5fb-4225-bf9b-f1fa26ab97b2
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: 'Created pod: isvc-canary-rollback-predictor-97977b6b5-cp7gp'
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-predictor-97977b6b5.18d63772aa41399e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14954"
    uid: ae521bbc-3e24-449e-89ad-d6e8a8014254
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:00Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-97977b6b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15944"
    uid: bb9a4eaf-f5fb-4225-bf9b-f1fa26ab97b2
  kind: Event
  lastTimestamp: "2026-09-17T21:03:00Z"
  message: 'Deleted pod: isvc-canary-rollback-predictor-97977b6b5-4dlsq'
  metadata:
    creationTimestamp: "2026-09-17T21:03:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:00Z"
    name: isvc-canary-rollback-predictor-97977b6b5.18d6378cf562ae2e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15947"
    uid: bddc099e-18f9-49ee-9b6c-5d13b1b96e17
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-97977b6b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15983"
    uid: bb9a4eaf-f5fb-4225-bf9b-f1fa26ab97b2
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: 'Created pod: isvc-canary-rollback-predictor-97977b6b5-lfppv'
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor-97977b6b5.18d6378d5fc1c077
    namespace: kserve-ci-e2e-test
    resourceVersion: "15998"
    uid: 530d5c17-ce9f-4013-9d75-cb39dbb93e11
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14912"
    uid: 22ab77c7-1e18-437e-a936-79b7639c8f6c
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-97977b6b5 from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-predictor.18d637729e8c7aa5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14916"
    uid: 10d00496-520d-40e0-bcbf-9086368b1049
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:00Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15940"
    uid: 22ab77c7-1e18-437e-a936-79b7639c8f6c
  kind: Event
  lastTimestamp: "2026-09-17T21:03:00Z"
  message: Scaled down replica set isvc-canary-rollback-predictor-97977b6b5 from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:03:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:00Z"
    name: isvc-canary-rollback-predictor.18d6378cf4c799ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "15945"
    uid: 2aae7477-dc04-4db4-ade8-910648abbbe1
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15976"
    uid: 22ab77c7-1e18-437e-a936-79b7639c8f6c
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-97977b6b5 from 1 to
    2
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-predictor.18d6378d5afb6c61
    namespace: kserve-ci-e2e-test
    resourceVersion: "15986"
    uid: e21f8069-96a0-4e5e-a0b8-7fb6027d28e0
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:01:07.367876Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14914"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d637729f7d94ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "14923"
    uid: ef72e356-df50-498e-8e63-ed2c01e9312d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14928"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: Add eth0 [10.133.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63772c130a71a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14986"
    uid: b0397753-dbaf-40bc-9481-05329e977c63
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63772c2b8de93
    namespace: kserve-ci-e2e-test
    resourceVersion: "14988"
    uid: 0c2e9fae-68ba-4952-9e52-29f7ec288c3f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63772c61ff2aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "14991"
    uid: f87db5cc-0d6f-483b-b21e-181e422c9aa0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63772c69bba76
    namespace: kserve-ci-e2e-test
    resourceVersion: "14993"
    uid: 7520a7a8-f859-4f51-9d90-e714bbe87a30
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:12Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
  metadata:
    creationTimestamp: "2026-09-17T21:01:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:12Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63773bbfe40d9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15049"
    uid: b665cfe3-a38a-4b59-a91a-3dae0cfb61d7
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    in 19.062s (19.062s including waiting). Image size: 1604543374 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d637782c31dfd4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15207"
    uid: c02dfed5-3fab-44f7-a43c-ac5befb11800
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d6377830647142
    namespace: kserve-ci-e2e-test
    resourceVersion: "15210"
    uid: 84755765-88e7-4ab6-8531-b318bf37c789
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d6377830d9e9eb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15211"
    uid: 6c29b106-780e-4c70-9933-ee73c031430a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d6377830e6c0f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15212"
    uid: c7eee443-6f7a-44a5-833a-3f6e22db9a7c
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.916s (2.916s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63778deb8450e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15247"
    uid: 489a4d0c-61dc-4194-a4aa-6d5ea48a5b71
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63778e14f2de5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15249"
    uid: f79b6804-03e8-4528-aacd-04f8769b70fc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63778e1cce7b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15251"
    uid: 86aa37bc-35bc-418a-9970-558169ab0003
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:02:50Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:02:50Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d63778e6a7ee1c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15835"
    uid: ef07e78f-ddc2-4c20-8097-472727881800
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d6378d5ba2fafa
    namespace: kserve-ci-e2e-test
    resourceVersion: "15991"
    uid: b36d8f10-d530-42f3-8c04-9bed12e29ae5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d6378d5ba38cbf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15992"
    uid: fc0b892b-edce-4bd6-94ae-a9fd90899fb5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: e7d49a27-d465-49af-8f11-11d586d53a21
  kind: Event
  lastTimestamp: "2026-09-17T21:03:05Z"
  message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp
    10.133.0.25:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:03:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:05Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2.18d6378e1726a8b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16041"
    uid: 5b08e2b1-ff53-44a9-a048-912ee0da45f7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-v2-predictor-5db5496f8d
    namespace: kserve-ci-e2e-test
    resourceVersion: "14899"
    uid: 81f6bfb7-fae3-48fb-bc37-1b7e38fb6c33
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: 'Created pod: isvc-canary-rollback-v2-predictor-5db5496f8d-czkk2'
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-v2-predictor-5db5496f8d.18d637729ed30424
    namespace: kserve-ci-e2e-test
    resourceVersion: "14917"
    uid: 5aa6e97b-bf37-42a4-9af9-b9de998c3f02
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14898"
    uid: 2eb8f33e-fec9-490d-9cf8-d415789e9ecf
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: Scaled up replica set isvc-canary-rollback-v2-predictor-5db5496f8d from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-canary-rollback-v2-predictor.18d63772960de82c
    namespace: kserve-ci-e2e-test
    resourceVersion: "14901"
    uid: 234cfe68-12d9-425d-bb23-2f8bf698e7dd
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 20
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "14887"
    uid: 83eda0b3-a213-4704-9399-514dc7341a3b
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback.18d637729e48ffc0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16002"
    uid: fd749015-9eab-4783-9605-7f4541bbc1f2
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "14887"
    uid: 83eda0b3-a213-4704-9399-514dc7341a3b
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: 'Failed to update status for InferenceService "isvc-canary-rollback": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback.18d637729ee0a1a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15995"
    uid: d23d180c-ab06-4c5a-8e2b-4577db7c5fd7
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "14887"
    uid: 83eda0b3-a213-4704-9399-514dc7341a3b
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback.18d637729ee102a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15996"
    uid: 9829686f-a239-4254-9a9e-d0133013e35e
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:02:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "15861"
    uid: 83eda0b3-a213-4704-9399-514dc7341a3b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:21Z"
  message: InferenceService [isvc-canary-rollback] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:02:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:21Z"
    name: isvc-canary-rollback.18d6378acbc5b728
    namespace: kserve-ci-e2e-test
    resourceVersion: "16775"
    uid: f3fd6a91-9ff8-4c67-af9c-0a261d85f0b6
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "16004"
    uid: 83eda0b3-a213-4704-9399-514dc7341a3b
  kind: Event
  lastTimestamp: "2026-09-17T21:03:02Z"
  message: 'InferenceService [isvc-canary-rollback] is no longer Ready because of:
    PredictorReady, Ready, Stopped'
  metadata:
    creationTimestamp: "2026-09-17T21:03:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:02Z"
    name: isvc-canary-rollback.18d6378d61181e56
    namespace: kserve-ci-e2e-test
    resourceVersion: "16007"
    uid: 18f36939-bff2-4ed2-9ba7-cdf9a83afadc
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:04:27.761206Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16880"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-55656fdf77-5bm4n
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a147dc0818
    namespace: kserve-ci-e2e-test
    resourceVersion: "16883"
    uid: 9784cb1e-642e-4857-b133-1a86d33f4c2c
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: Add eth0 [10.132.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a169dcdd21
    namespace: kserve-ci-e2e-test
    resourceVersion: "16921"
    uid: b9b4d558-bfc5-430a-97bc-b8fa116516a8
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a16c1e2ff5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16923"
    uid: ce972d04-4b1b-4e1c-b4d4-ca5f478507cb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a16f099f06
    namespace: kserve-ci-e2e-test
    resourceVersion: "16926"
    uid: a24e5963-feaa-41aa-853f-acd26bb14703
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a16f7ef55e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16927"
    uid: e77db251-500c-4ea0-a53b-679eba70a19d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a277490a5a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16986"
    uid: ca4a8e5c-1ed7-4117-a4a5-fcf954ca19b0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:04:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a27a71e4a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16988"
    uid: 2793177b-448a-4f9d-96c9-d047ee9675dd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:04:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a27ae57c1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16989"
    uid: 66b7cc3b-70ab-44a9-b99a-c33ed2d12b2d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a27aefb49e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16991"
    uid: d3f07009-9f3f-4f50-a14a-d7a1c0eb5c55
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:04:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a27e09b738
    namespace: kserve-ci-e2e-test
    resourceVersion: "16993"
    uid: 7c349c77-7c63-4b3d-b597-8eaa968fdaff
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:04:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a27e7bd137
    namespace: kserve-ci-e2e-test
    resourceVersion: "16994"
    uid: 57cd0918-b2cd-4793-9797-91aaa5ba2b0e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:05:44Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:44Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637a2b3520afb
    namespace: kserve-ci-e2e-test
    resourceVersion: "17797"
    uid: fd069009-9461-417e-b2ad-10c8e09d0010
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:06:07Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:06:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:07Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637b885991e4c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18040"
    uid: c431d5bb-faa6-4ef7-a111-7b23addc4a0d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:06:07Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:06:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:07Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637b88599b287
    namespace: kserve-ci-e2e-test
    resourceVersion: "18041"
    uid: 449966e6-1335-4d37-882e-bdd0f3b27f69
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: 395c4884-f6d1-450a-9db9-1073b32e5a53
  kind: Event
  lastTimestamp: "2026-09-17T21:06:09Z"
  message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp
    10.132.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:06:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:09Z"
    name: isvc-canary-stop-predictor-55656fdf77-5bm4n.18d637b90d984ba3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18056"
    uid: aff8c96e-f12b-4ff7-9928-ecb7306aa511
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:04:27.874592Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16891"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-55656fdf77-hm692
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a14e9e2f37
    namespace: kserve-ci-e2e-test
    resourceVersion: "16897"
    uid: f4b84a2d-bab2-4121-a024-b05b9f2638bd
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:29Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:29Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : failed to sync secret
    cache: timed out waiting for the condition'
  metadata:
    creationTimestamp: "2026-09-17T21:04:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:29Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a19550b49c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16939"
    uid: f2d449fd-b555-4bfb-8902-35e893819a6d
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:29Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16915"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:29Z"
  message: Add eth0 [10.133.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:04:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:04:29Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a1c9daf910
    namespace: kserve-ci-e2e-test
    resourceVersion: "16953"
    uid: 07bbbba3-741f-4610-8b06-ee39d96a416f
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:29Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:29Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a1cb513fe0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16955"
    uid: ab20a729-473a-4f13-8d80-0d4e19a82d7c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:30Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:04:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:30Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a1ce62b97a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16957"
    uid: 1e884fc3-48e9-4040-9d88-9421486991f6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:30Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:04:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:30Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a1cee5f14a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16958"
    uid: 0e86c622-7093-4c8e-af3f-6ca1195a24ae
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:34Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:34Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2c1f34b4a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17014"
    uid: 0bccde42-6217-4664-a034-849153f514e8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:34Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:34Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2c4e67897
    namespace: kserve-ci-e2e-test
    resourceVersion: "17017"
    uid: 9f61dc09-425a-456b-b2b8-58c64d833009
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:34Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:34Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2c555c8f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17018"
    uid: 1817e8a4-4a08-43d9-8815-8ca46e049ddc
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:34Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:34Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2c55e56a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17019"
    uid: 5fd3b372-fbe4-4473-a35e-9b2dedad5352
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:34Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:34Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2c81e81a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17021"
    uid: 749bbba8-0394-43b5-9156-4dba2828a884
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:34Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:34Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2c88ca917
    namespace: kserve-ci-e2e-test
    resourceVersion: "17022"
    uid: 3eed5b2c-c322-4bb5-a627-dba39d64a55f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:05:51Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:04:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:51Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637a2fdf364ac
    namespace: kserve-ci-e2e-test
    resourceVersion: "17852"
    uid: 80074f69-3a51-493f-9569-4ecfd1226d2f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:06:04Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:06:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:04Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637b7d3476f04
    namespace: kserve-ci-e2e-test
    resourceVersion: "17988"
    uid: ec09ed43-8181-4d01-8c97-1e608c9f38fa
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:06:04Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:06:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:04Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637b7d347ebd6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17990"
    uid: d29b89b4-a66a-4bca-af84-6ecc02e68be4
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-55656fdf77-hm692
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 77dcbaf7-6ebd-45c9-87c0-c1fb3ab31cc9
  kind: Event
  lastTimestamp: "2026-09-17T21:06:06Z"
  message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp
    10.133.0.30:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:06:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:06Z"
    name: isvc-canary-stop-predictor-55656fdf77-hm692.18d637b82e34debd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18006"
    uid: 7f64f222-367b-4e57-9cab-0a990e4e37f0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-55656fdf77
    namespace: kserve-ci-e2e-test
    resourceVersion: "16862"
    uid: b78a4235-4fc6-42b5-b30f-b2493975d8e0
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: 'Created pod: isvc-canary-stop-predictor-55656fdf77-5bm4n'
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-predictor-55656fdf77.18d637a147349ae7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16882"
    uid: 07e201ea-59f4-4943-975c-a6139305fae5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-55656fdf77
    namespace: kserve-ci-e2e-test
    resourceVersion: "16862"
    uid: b78a4235-4fc6-42b5-b30f-b2493975d8e0
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: 'Created pod: isvc-canary-stop-predictor-55656fdf77-hm692'
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-predictor-55656fdf77.18d637a14d7997c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16892"
    uid: f109acfa-34cc-4580-a365-e85876ded187
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:04Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-55656fdf77
    namespace: kserve-ci-e2e-test
    resourceVersion: "17981"
    uid: b78a4235-4fc6-42b5-b30f-b2493975d8e0
  kind: Event
  lastTimestamp: "2026-09-17T21:06:04Z"
  message: 'Deleted pod: isvc-canary-stop-predictor-55656fdf77-hm692'
  metadata:
    creationTimestamp: "2026-09-17T21:06:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:04Z"
    name: isvc-canary-stop-predictor-55656fdf77.18d637b7d31b536f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17984"
    uid: b47fd084-064d-45f1-b9ef-b4c19ddbc216
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16861"
    uid: 18d21404-9555-4ac5-a368-68402266202c
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: Scaled up replica set isvc-canary-stop-predictor-55656fdf77 from 0 to 2
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-predictor.18d637a13ee06085
    namespace: kserve-ci-e2e-test
    resourceVersion: "16867"
    uid: 5a952392-29f4-403e-88f3-43206166ee0a
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:04Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17976"
    uid: 18d21404-9555-4ac5-a368-68402266202c
  kind: Event
  lastTimestamp: "2026-09-17T21:06:04Z"
  message: Scaled down replica set isvc-canary-stop-predictor-55656fdf77 from 2 to
    1
  metadata:
    creationTimestamp: "2026-09-17T21:06:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:04Z"
    name: isvc-canary-stop-predictor.18d637b7d280765d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17982"
    uid: 2a0ff067-0b87-4a68-bf19-f8fab3b582ae
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:04:27.661235Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16868"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a141e68336
    namespace: kserve-ci-e2e-test
    resourceVersion: "16876"
    uid: 53af27ed-f4e3-447d-8155-0b6034f81780
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a14d9a2d53
    namespace: kserve-ci-e2e-test
    resourceVersion: "16895"
    uid: 6619cd0e-346c-4604-95ff-fb208eb3a2a2
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16879"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: Add eth0 [10.133.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a18755f56d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16929"
    uid: b8d6b205-ff60-4329-a8fc-8fbc9dc124e5
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a18a61d922
    namespace: kserve-ci-e2e-test
    resourceVersion: "16932"
    uid: d0ede6b2-430f-484c-bc04-25748b28c0dc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a18d3bf6b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16935"
    uid: 3d4fd5fe-04f1-4d09-9a65-ef4bef00525f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:28Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a18daaf84d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16936"
    uid: 8ef97b70-443e-4700-9b4a-4895f0977d54
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:33Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:33Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a285e47409
    namespace: kserve-ci-e2e-test
    resourceVersion: "16996"
    uid: 1daeb26a-e46b-4931-86a8-b82e53948f6f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:33Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:33Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a28923519c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16999"
    uid: f76ce8f0-efd3-4178-aa66-41b96025e1c1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:33Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:33Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a289957cec
    namespace: kserve-ci-e2e-test
    resourceVersion: "17000"
    uid: 81b9fd7c-0784-4878-ad53-d7514d202775
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:33Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:33Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a289a07fae
    namespace: kserve-ci-e2e-test
    resourceVersion: "17001"
    uid: ed89e07b-544f-4909-9024-dce02a0b2632
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:33Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:33Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a28c646393
    namespace: kserve-ci-e2e-test
    resourceVersion: "17005"
    uid: 9387b521-b4c8-4c96-8a2b-d4b518245e7d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:33Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:04:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:33Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a28ce6df50
    namespace: kserve-ci-e2e-test
    resourceVersion: "17006"
    uid: bd511e3a-337f-436d-a07b-f092d4991588
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:05:54Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:04:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:54Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637a2c1e95a3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17888"
    uid: c80b19c9-96a2-4fd1-9a92-029b17a55421
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:06:07Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:06:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:07Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637b8839b8ea7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18030"
    uid: 37bb65ec-8663-4b3c-a87f-4268c9ecc727
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:06:07Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:06:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:07Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637b8839c9e41
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 78838486-b9f8-499f-bf13-33e9013dd18f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16869"
    uid: 08b521ce-377f-4f24-ba4a-95585aca48d5
  kind: Event
  lastTimestamp: "2026-09-17T21:06:10Z"
  message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp
    10.133.0.29:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:06:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:10Z"
    name: isvc-canary-stop-v2-predictor-d65d59944-ww6pd.18d637b91c5278a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18060"
    uid: cc867e0f-2c46-407d-954e-5ae001b77538
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-v2-predictor-d65d59944
    namespace: kserve-ci-e2e-test
    resourceVersion: "16849"
    uid: 8c1064cf-7392-41ae-9e8d-5b788d49f2e7
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: 'Created pod: isvc-canary-stop-v2-predictor-d65d59944-ww6pd'
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-v2-predictor-d65d59944.18d637a140a056ba
    namespace: kserve-ci-e2e-test
    resourceVersion: "16870"
    uid: 57119580-eea4-47af-a5ea-9b1643e1fde5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16845"
    uid: 1003f280-fe1e-42e6-bac4-ae3f484a568f
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: Scaled up replica set isvc-canary-stop-v2-predictor-d65d59944 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop-v2-predictor.18d637a13912e2b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16851"
    uid: 20a4b0de-58f7-483e-b8e6-a6ee3c5cf1e1
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 21
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 566e312c-7aed-4f51-9a28-4fb98d96eafd
  kind: Event
  lastTimestamp: "2026-09-17T21:06:04Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:04Z"
    name: isvc-canary-stop.18d637a13ee383d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17996"
    uid: 97259a8a-5bb4-45d5-b343-5d31e6462562
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 566e312c-7aed-4f51-9a28-4fb98d96eafd
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:27Z"
    name: isvc-canary-stop.18d637a140b44394
    namespace: kserve-ci-e2e-test
    resourceVersion: "16908"
    uid: e61410d2-d29f-45ab-ab63-843db4367d7e
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:27Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 566e312c-7aed-4f51-9a28-4fb98d96eafd
  kind: Event
  lastTimestamp: "2026-09-17T21:04:27Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:04:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:28Z"
    name: isvc-canary-stop.18d637a140b48a84
    namespace: kserve-ci-e2e-test
    resourceVersion: "16909"
    uid: a058de82-ab6d-41b9-a3e4-08e41dbc18b7
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "17940"
    uid: 566e312c-7aed-4f51-9a28-4fb98d96eafd
  kind: Event
  lastTimestamp: "2026-09-17T21:06:01Z"
  message: InferenceService [isvc-canary-stop] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:06:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:01Z"
    name: isvc-canary-stop.18d637b70a644052
    namespace: kserve-ci-e2e-test
    resourceVersion: "17941"
    uid: 9a136f9b-6e10-4b2b-bef4-e08119574c5d
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:08:24.352626Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    namespace: kserve-ci-e2e-test
    resourceVersion: "19528"
    uid: 8f6f4469-dbdb-45cb-adfc-d10762ef5ec6
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:08:24Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx.18d637d85dce9ec3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19535"
    uid: 0332ca68-1d60-4f74-8828-178021a3a6b1
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    namespace: kserve-ci-e2e-test
    resourceVersion: "19541"
    uid: 8f6f4469-dbdb-45cb-adfc-d10762ef5ec6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:24Z"
  message: Add eth0 [10.133.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:08:24Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx.18d637d87fbb4038
    namespace: kserve-ci-e2e-test
    resourceVersion: "19549"
    uid: 8724ddd8-2e25-4199-9775-05184aaaa3d2
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    namespace: kserve-ci-e2e-test
    resourceVersion: "19530"
    uid: 8f6f4469-dbdb-45cb-adfc-d10762ef5ec6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:29Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:29Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx.18d637d881116ed3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19593"
    uid: 390a8933-3cee-4856-849e-9687f3e7e3af
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    namespace: kserve-ci-e2e-test
    resourceVersion: "19530"
    uid: 8f6f4469-dbdb-45cb-adfc-d10762ef5ec6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:29Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:29Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx.18d637d883dca7e2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19596"
    uid: 10c12e40-423c-47d3-bc1b-d96038f7b44c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    namespace: kserve-ci-e2e-test
    resourceVersion: "19530"
    uid: 8f6f4469-dbdb-45cb-adfc-d10762ef5ec6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:29Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:08:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:29Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx.18d637d8844c0caa
    namespace: kserve-ci-e2e-test
    resourceVersion: "19599"
    uid: 947138c5-faa3-4659-8c7d-2780764d1c82
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx
    namespace: kserve-ci-e2e-test
    resourceVersion: "19530"
    uid: 8f6f4469-dbdb-45cb-adfc-d10762ef5ec6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:34Z"
  message: Stopping container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:08:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:34Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx.18d637daaf45137f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19655"
    uid: 82a3508f-806e-4bb8-967c-ca747318b7c0
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-init-fail-042be1-predictor-55867d7b6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19520"
    uid: 29ea0ff7-5d35-4fb3-848f-f9e9e0f3a48b
  kind: Event
  lastTimestamp: "2026-09-17T21:08:24Z"
  message: 'Created pod: isvc-init-fail-042be1-predictor-55867d7b6d-6dwjx'
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:24Z"
    name: isvc-init-fail-042be1-predictor-55867d7b6d.18d637d85d44f738
    namespace: kserve-ci-e2e-test
    resourceVersion: "19529"
    uid: f969691f-4aab-4c34-9a3d-2fb9929a5f66
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-init-fail-042be1-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19518"
    uid: f499e33c-e6ea-4bcb-95f7-7b3295dd414f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:24Z"
  message: Scaled up replica set isvc-init-fail-042be1-predictor-55867d7b6d from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:24Z"
    name: isvc-init-fail-042be1-predictor.18d637d8599ecce7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19521"
    uid: 7e978b0a-ccf7-472f-980a-9e30716a4352
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-042be1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19509"
    uid: 461a6c8a-591c-4249-b504-8f4f5490745f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:29Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:29Z"
    name: isvc-init-fail-042be1.18d637d859a1a31c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19600"
    uid: 0ee4a62a-647e-4119-b058-d5468d35927d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-042be1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19509"
    uid: 461a6c8a-591c-4249-b504-8f4f5490745f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:24Z"
  message: 'Failed to update status for InferenceService "isvc-init-fail-042be1":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-042be1":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:24Z"
    name: isvc-init-fail-042be1.18d637d85a32c4d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19524"
    uid: b196da0c-b3e6-44b7-b687-34b0d9218547
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:24Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-042be1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19509"
    uid: 461a6c8a-591c-4249-b504-8f4f5490745f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:24Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-init-fail-042be1": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:24Z"
    name: isvc-init-fail-042be1.18d637d85a330373
    namespace: kserve-ci-e2e-test
    resourceVersion: "19526"
    uid: 8c6265a8-dea6-427c-b7a5-ea3804bcc364
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:07:13.531623Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18813"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:07:13Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c7e08bcb58
    namespace: kserve-ci-e2e-test
    resourceVersion: "18817"
    uid: ba4a0cc7-2376-453a-b147-fd29ee4c46a4
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:13Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:13Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-8d8d6-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:13Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c7eb0cf9f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18829"
    uid: bcbbfd64-850b-4f87-a146-9a50400cf09e
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:14Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18821"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:14Z"
  message: Add eth0 [10.133.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:07:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:07:14Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c8265d701f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18837"
    uid: 2c3cef70-685c-4563-839f-d1b92fc3e0f1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:14Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:07:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:14Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c82805180c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18839"
    uid: 0aa9176a-e317-4b6a-8099-aff5e7a0e47e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:14Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:07:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:14Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c82aef3b6a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18841"
    uid: 55516b90-82e2-4786-b241-8bce5a3190e3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:14Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:07:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:14Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c82b5f900d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18842"
    uid: 4b75fd82-081b-4a2c-892b-99c80d35dc83
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c91bfa8b9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18885"
    uid: 6d840dda-5055-435f-b5b7-2ebb5211b2de
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c91f0def96
    namespace: kserve-ci-e2e-test
    resourceVersion: "18887"
    uid: 97788edb-ef1a-431f-ae1d-f6ebdb26f3b9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c91f7efcb1
    namespace: kserve-ci-e2e-test
    resourceVersion: "18890"
    uid: 440d8ec0-dcf6-4195-bbf3-cc0adeed84a2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c91f895760
    namespace: kserve-ci-e2e-test
    resourceVersion: "18891"
    uid: ecea408b-21e8-4f19-ba9b-37b17e1934ca
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c922707341
    namespace: kserve-ci-e2e-test
    resourceVersion: "18893"
    uid: 6896fb51-5e3d-4f2b-9858-cc32f3ec7b93
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c922df3933
    namespace: kserve-ci-e2e-test
    resourceVersion: "18894"
    uid: d38b6144-1cd8-4edc-bcd3-ae5a05c64684
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: Container image "quay.io/opendatahub/kserve-agent@sha256:b59b759dbc00d731ff1eeb0e6abaa9d6adb19d1c613e234f3ace5c825641711a"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c922ea4d31
    namespace: kserve-ci-e2e-test
    resourceVersion: "18895"
    uid: 93033b02-c1d1-425e-af69-b9b5fb9ec35d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c9257f2c92
    namespace: kserve-ci-e2e-test
    resourceVersion: "18896"
    uid: 064133b7-b071-4d8d-a7e6-cf1783827db9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:18Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:18Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c925ed9c51
    namespace: kserve-ci-e2e-test
    resourceVersion: "18897"
    uid: 8574a7d2-7850-451c-8b49-89dac8261ebe
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:08:56Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:07:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:56Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c993d0ca32
    namespace: kserve-ci-e2e-test
    resourceVersion: "20000"
    uid: 252fa647-c4a6-4ffc-855d-3ad8c77a4684
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:08:56Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T21:07:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:56Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637c993db492f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20001"
    uid: 8054ff24-d872-4ab7-bc24-d44f4f688790
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637de24c3ba9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19858"
    uid: a1b1b3f2-8e78-4b3e-89ba-b5885ff9ed3c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637de24c3cecf
    namespace: kserve-ci-e2e-test
    resourceVersion: "19861"
    uid: 4f4ce64e-c59f-4434-b1e6-9015fcaa78c3
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637de24c43d13
    namespace: kserve-ci-e2e-test
    resourceVersion: "19866"
    uid: ea64a80c-e01a-45e8-838f-c51dede95550
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 46cdee62-f1b1-4a8b-bd3c-0717fb883fe9
  kind: Event
  lastTimestamp: "2026-09-17T21:09:06Z"
  message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp
    10.133.0.33:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:08:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:09:06Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w.18d637dec40cbcf8
    namespace: kserve-ci-e2e-test
    resourceVersion: "20082"
    uid: 59691e4d-1b3c-4b76-b648-bb11a06c703d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:13Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18803"
    uid: cbbfdabc-fab5-48f8-add4-0120820a4f40
  kind: Event
  lastTimestamp: "2026-09-17T21:07:13Z"
  message: 'Created pod: isvc-logger-raw-8d8d6-predictor-78c477c7b7-zd86w'
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:13Z"
    name: isvc-logger-raw-8d8d6-predictor-78c477c7b7.18d637c7dff0fbac
    namespace: kserve-ci-e2e-test
    resourceVersion: "18814"
    uid: 8e2f5f62-88f7-4e22-8ecc-b7ce506177b2
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:13Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18800"
    uid: 8d5c859c-8c98-4899-88ae-aee142af830e
  kind: Event
  lastTimestamp: "2026-09-17T21:07:13Z"
  message: Scaled up replica set isvc-logger-raw-8d8d6-predictor-78c477c7b7 from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:13Z"
    name: isvc-logger-raw-8d8d6-predictor.18d637c7dca5752d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18805"
    uid: 09cdb2fb-9e49-466b-b4b6-fe9a8f206f68
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:28Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18801"
    uid: 3b4da444-31c3-4eb2-8099-c6966c2c4233
  kind: Event
  lastTimestamp: "2026-09-17T21:07:43Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:07:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:43Z"
    name: isvc-logger-raw-8d8d6-predictor.18d637cb5b0deffb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19097"
    uid: ef36e7a9-19dc-4024-b955-ea769c404f48
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:28Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18801"
    uid: 3b4da444-31c3-4eb2-8099-c6966c2c4233
  kind: Event
  lastTimestamp: "2026-09-17T21:07:43Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:07:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:43Z"
    name: isvc-logger-raw-8d8d6-predictor.18d637cb5b76e70b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19098"
    uid: 35a8aa1f-97c4-4e19-9297-39bde56e3249
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:58Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18975"
    uid: 3b4da444-31c3-4eb2-8099-c6966c2c4233
  kind: Event
  lastTimestamp: "2026-09-17T21:08:43Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:07:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:43Z"
    name: isvc-logger-raw-8d8d6-predictor.18d637d2590071de
    namespace: kserve-ci-e2e-test
    resourceVersion: "19778"
    uid: 2f8dd703-1719-4867-8db1-52951f30bc07
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:58Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18975"
    uid: 3b4da444-31c3-4eb2-8099-c6966c2c4233
  kind: Event
  lastTimestamp: "2026-09-17T21:08:43Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:07:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:43Z"
    name: isvc-logger-raw-8d8d6-predictor.18d637d259728d91
    namespace: kserve-ci-e2e-test
    resourceVersion: "19779"
    uid: 1ebc2f43-6c95-41db-9c38-30b3c88ebf1c
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:13Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18793"
    uid: fc157455-e497-4f3d-be50-3cb88682fb16
  kind: Event
  lastTimestamp: "2026-09-17T21:08:36Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:36Z"
    name: isvc-logger-raw-8d8d6.18d637c7dc77d51b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19686"
    uid: 48b0e0f7-5f68-4905-a836-33beb6606873
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:13Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18793"
    uid: fc157455-e497-4f3d-be50-3cb88682fb16
  kind: Event
  lastTimestamp: "2026-09-17T21:07:13Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-8d8d6":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-8d8d6":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:13Z"
    name: isvc-logger-raw-8d8d6.18d637c7dcd1f06c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18823"
    uid: a5d9cbef-48c4-4b23-836d-dcc4f20f9364
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:13Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18795"
    uid: fc157455-e497-4f3d-be50-3cb88682fb16
  kind: Event
  lastTimestamp: "2026-09-17T21:07:13Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-logger-raw-8d8d6": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:07:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:13Z"
    name: isvc-logger-raw-8d8d6.18d637c7e2342dc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18824"
    uid: 46b2294d-2d09-4022-b299-b9d00ed3793a
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19684"
    uid: fc157455-e497-4f3d-be50-3cb88682fb16
  kind: Event
  lastTimestamp: "2026-09-17T21:08:36Z"
  message: InferenceService [isvc-logger-raw-8d8d6] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:08:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:36Z"
    name: isvc-logger-raw-8d8d6.18d637db4b6dc338
    namespace: kserve-ci-e2e-test
    resourceVersion: "19685"
    uid: 5f8e7d2a-23cf-48d2-a391-18247508841a
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:17:01.130423Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24142"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63850b029f90a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24146"
    uid: bfcd94aa-fea0-4860-ae5d-6c44ebf74fcd
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24151"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: Add eth0 [10.133.0.41/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63850d1f3563b
    namespace: kserve-ci-e2e-test
    resourceVersion: "24165"
    uid: 1fb48614-3ee1-4821-97c1-983541275217
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63850d35509f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24167"
    uid: be156f06-dc9c-425f-bb8b-a0d53b88026a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63850d60fa642
    namespace: kserve-ci-e2e-test
    resourceVersion: "24169"
    uid: 1c3547c7-1e3e-4ff8-acab-8356bc9716b2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63850d67c52f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "24170"
    uid: 45ef73b4-31cf-4c52-8730-97dc1fbfcda2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851deb87e84
    namespace: kserve-ci-e2e-test
    resourceVersion: "24207"
    uid: 1d314158-d1c2-405b-859e-10a85117ceb9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e1f0c0f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "24209"
    uid: 1f2cecb2-d058-4dac-940b-b7d72ada3dc1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e25cbeff
    namespace: kserve-ci-e2e-test
    resourceVersion: "24210"
    uid: 6ca5b221-392d-4718-b70f-6692253e47ed
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e266e0d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "24211"
    uid: 066895da-3484-4813-87c4-391e76ed7e5f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e4fba6e9
    namespace: kserve-ci-e2e-test
    resourceVersion: "24214"
    uid: f3921ea2-80b8-4128-8730-8a711800b748
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e5671c97
    namespace: kserve-ci-e2e-test
    resourceVersion: "24215"
    uid: 4166cc93-9a65-48e3-b812-8959f071b0db
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: Container image "quay.io/opendatahub/kserve-agent@sha256:b59b759dbc00d731ff1eeb0e6abaa9d6adb19d1c613e234f3ace5c825641711a"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e571538b
    namespace: kserve-ci-e2e-test
    resourceVersion: "24216"
    uid: 2ee4d296-766a-49ff-9b3c-cdec33dd768e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e84f981d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24218"
    uid: 1720b952-48fe-4e39-809f-261cf7e6803a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63851e8b93c28
    namespace: kserve-ci-e2e-test
    resourceVersion: "24220"
    uid: efa800c5-043b-418e-9621-bce551ca1946
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:51Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:51Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d638521abf6e14
    namespace: kserve-ci-e2e-test
    resourceVersion: "25061"
    uid: 0daacca2-50e2-43ca-b855-4a8b028f1e65
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:41Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T21:17:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:41Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d6385256960c9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24984"
    uid: c6a42887-f5a7-47dd-8f0d-105d1ebdf115
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:36Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:18:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:36Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63866fd66f6c9
    namespace: kserve-ci-e2e-test
    resourceVersion: "24900"
    uid: b4a68285-5c4f-422f-816c-96f4f9b4d7bc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:36Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-17T21:18:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:36Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63866fd67462e
    namespace: kserve-ci-e2e-test
    resourceVersion: "24901"
    uid: 3877ed7b-1e4f-4b36-ba2f-8daea840222e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:36Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:18:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:36Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d63866fd67a39f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24903"
    uid: 9475726b-aec0-4450-bcb2-e31d9f83331b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:39Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w
    namespace: kserve-ci-e2e-test
    resourceVersion: "24144"
    uid: a68d8b52-888b-4113-972c-3fc6aa9c7dc8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:49Z"
  message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp
    10.133.0.41:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:18:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:49Z"
    name: isvc-logger-raw-cipn-predictor-797ff577-vsv2w.18d6386786ccdaae
    namespace: kserve-ci-e2e-test
    resourceVersion: "25042"
    uid: 36469b07-b2a9-4470-92f0-cf71bb9be997
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-cipn-predictor-797ff577
    namespace: kserve-ci-e2e-test
    resourceVersion: "24131"
    uid: 64963fe6-2cdb-42d7-b10b-5d1a0bf29c9d
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: 'Created pod: isvc-logger-raw-cipn-predictor-797ff577-vsv2w'
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor-797ff577.18d63850af9a97cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24143"
    uid: 2f8fb297-e940-47d5-852c-e5df1844f142
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24129"
    uid: 1260a39b-2eb1-46f8-ba3e-91b513f0aaca
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: Scaled up replica set isvc-logger-raw-cipn-predictor-797ff577 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn-predictor.18d63850ab7b6b36
    namespace: kserve-ci-e2e-test
    resourceVersion: "24134"
    uid: 35b7ed5b-0576-4966-a783-8fe9da8f9d95
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24130"
    uid: 54efda22-5ee6-423a-9cdd-5785ee204417
  kind: Event
  lastTimestamp: "2026-09-17T21:17:46Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:17:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:46Z"
    name: isvc-logger-raw-cipn-predictor.18d638542a6a89f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "24513"
    uid: 8e68ef4d-8e6a-40d2-b9d4-04a1da647752
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24130"
    uid: 54efda22-5ee6-423a-9cdd-5785ee204417
  kind: Event
  lastTimestamp: "2026-09-17T21:17:46Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:17:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:46Z"
    name: isvc-logger-raw-cipn-predictor.18d638542ad5b78a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24515"
    uid: b1282498-357b-4d93-8f27-e7d1fc6eb3bf
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:01Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24299"
    uid: 54efda22-5ee6-423a-9cdd-5785ee204417
  kind: Event
  lastTimestamp: "2026-09-17T21:18:31Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:18:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:18:31Z"
    name: isvc-logger-raw-cipn-predictor.18d6385ea889b71f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24839"
    uid: 5bc7f88c-9f0b-47b8-90f8-09829949f32f
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:01Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24299"
    uid: 54efda22-5ee6-423a-9cdd-5785ee204417
  kind: Event
  lastTimestamp: "2026-09-17T21:18:31Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:18:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:18:31Z"
    name: isvc-logger-raw-cipn-predictor.18d6385ea8febaab
    namespace: kserve-ci-e2e-test
    resourceVersion: "24841"
    uid: c7aae23d-abe7-4cb8-81a8-a18ca712610b
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 13
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24121"
    uid: ee9fa31e-4f44-4532-9170-a42acc3426a8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:21Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:18:21Z"
    name: isvc-logger-raw-cipn.18d63850ab7a7713
    namespace: kserve-ci-e2e-test
    resourceVersion: "24773"
    uid: e32539d0-1394-44ff-aecc-529358cd58a2
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24121"
    uid: ee9fa31e-4f44-4532-9170-a42acc3426a8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-cipn": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn.18d63850ac1161d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "24152"
    uid: 13715b60-549f-45a0-bac0-859a32e8786f
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24124"
    uid: ee9fa31e-4f44-4532-9170-a42acc3426a8
  kind: Event
  lastTimestamp: "2026-09-17T21:17:01Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:17:01Z"
    name: isvc-logger-raw-cipn.18d63850b2265258
    namespace: kserve-ci-e2e-test
    resourceVersion: "24153"
    uid: 9123efca-5900-460e-94ea-1d63f59d22de
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:21Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24770"
    uid: ee9fa31e-4f44-4532-9170-a42acc3426a8
  kind: Event
  lastTimestamp: "2026-09-17T21:18:21Z"
  message: InferenceService [isvc-logger-raw-cipn] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:18:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:18:21Z"
    name: isvc-logger-raw-cipn.18d638636beb1e59
    namespace: kserve-ci-e2e-test
    resourceVersion: "24772"
    uid: 065484d6-5c25-438c-ad6b-e08d37455384
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:06:17.831633Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18151"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:06:17Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bae8917534
    namespace: kserve-ci-e2e-test
    resourceVersion: "18157"
    uid: 855ba704-0209-452d-931f-2c879c17bd85
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:17Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:17Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-b61317-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:17Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637baf0d65557
    namespace: kserve-ci-e2e-test
    resourceVersion: "18165"
    uid: 0484b81e-9472-433b-b353-2164e7b36010
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:19Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18161"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:19Z"
  message: Add eth0 [10.132.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:06:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:06:19Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bb2e6d9c6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18180"
    uid: beb422b9-3fb9-4456-91b5-dd5861d87585
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:19Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:06:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:19Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bb2fedd123
    namespace: kserve-ci-e2e-test
    resourceVersion: "18182"
    uid: e13f0acc-bd77-47d6-809d-bc938db6509e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:19Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:06:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:19Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bb32f1a135
    namespace: kserve-ci-e2e-test
    resourceVersion: "18184"
    uid: 1fd275d1-d1e3-41a5-b6bf-0e318cef20de
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:19Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:06:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:19Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bb336145e1
    namespace: kserve-ci-e2e-test
    resourceVersion: "18185"
    uid: ffdfe2b5-04b8-411a-8872-5fbb83c16ddc
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:23Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:06:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:23Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bc3182c68a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18223"
    uid: 06ca20b9-4610-4222-884b-ccdcbf39e51a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:23Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:06:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:23Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bc34646107
    namespace: kserve-ci-e2e-test
    resourceVersion: "18226"
    uid: 543aa2db-4c3d-46cd-9ce0-aa26be4b3b9d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:23Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:06:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:23Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bc34d8f2d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18227"
    uid: f949106d-0552-43e7-889a-c87bb37350da
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:23Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:06:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:23Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bc34e35f7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18228"
    uid: 7088c443-3042-4bf8-ab55-c218d1e78e72
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:23Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:06:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:23Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bc37fc2224
    namespace: kserve-ci-e2e-test
    resourceVersion: "18231"
    uid: d115fbc7-a83f-4dd2-a1c2-96c48981a8c0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:06:23Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:06:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:23Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bc3872ef9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18232"
    uid: 7c8706db-161a-47fe-b04f-9cedfee3e9a8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:08:21Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:21Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637bca95d5568
    namespace: kserve-ci-e2e-test
    resourceVersion: "19485"
    uid: e2bbd10d-1189-4eb7-8411-f48ba703be46
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:08:19Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:08:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:19Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637d728fe7e37
    namespace: kserve-ci-e2e-test
    resourceVersion: "19459"
    uid: ace4795a-6402-4cb8-8d55-57a385289c6f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:08:19Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:08:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:19Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637d728fedce5
    namespace: kserve-ci-e2e-test
    resourceVersion: "19461"
    uid: ebd7ff37-b527-4abc-9e14-38de0864896e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db
    namespace: kserve-ci-e2e-test
    resourceVersion: "18155"
    uid: 92420c11-81b3-44aa-ac53-f139e0ae5329
  kind: Event
  lastTimestamp: "2026-09-17T21:08:21Z"
  message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp
    10.132.0.29:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:21Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db.18d637d7abaeaf92
    namespace: kserve-ci-e2e-test
    resourceVersion: "19484"
    uid: dc78aaea-f1b6-4610-a738-2d0d1190ac36
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:17Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-primary-b61317-predictor-6d9bb74dbf
    namespace: kserve-ci-e2e-test
    resourceVersion: "18144"
    uid: 05f16121-d854-4bd4-90fc-02d8ef231713
  kind: Event
  lastTimestamp: "2026-09-17T21:06:17Z"
  message: 'Created pod: isvc-primary-b61317-predictor-6d9bb74dbf-cb8db'
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:17Z"
    name: isvc-primary-b61317-predictor-6d9bb74dbf.18d637bae8090e46
    namespace: kserve-ci-e2e-test
    resourceVersion: "18154"
    uid: 19206e24-fc40-4123-84ec-867a4dc9f829
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:17Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-primary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18142"
    uid: 889e83a1-0df0-4206-bdb7-9f8202d3f39f
  kind: Event
  lastTimestamp: "2026-09-17T21:06:17Z"
  message: Scaled up replica set isvc-primary-b61317-predictor-6d9bb74dbf from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:17Z"
    name: isvc-primary-b61317-predictor.18d637bae4c47c24
    namespace: kserve-ci-e2e-test
    resourceVersion: "18146"
    uid: 5b99fa91-7dfd-4f94-bfe0-df4d2b2d680a
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:32Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18143"
    uid: 107c6c52-6fa9-4a31-a7fd-e47f70095ee0
  kind: Event
  lastTimestamp: "2026-09-17T21:07:02Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:06:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:02Z"
    name: isvc-primary-b61317-predictor.18d637be6374d0fa
    namespace: kserve-ci-e2e-test
    resourceVersion: "18587"
    uid: 5deec0a2-6938-4234-b9a8-011e829a9c41
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:32Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18143"
    uid: 107c6c52-6fa9-4a31-a7fd-e47f70095ee0
  kind: Event
  lastTimestamp: "2026-09-17T21:07:02Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:06:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:02Z"
    name: isvc-primary-b61317-predictor.18d637be64c636eb
    namespace: kserve-ci-e2e-test
    resourceVersion: "18588"
    uid: ebb61471-f639-4c8f-99ef-9a5b935a2129
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:47Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18304"
    uid: 107c6c52-6fa9-4a31-a7fd-e47f70095ee0
  kind: Event
  lastTimestamp: "2026-09-17T21:08:02Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-primary-b61317-predictor.18d637c1e3f6d4a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19261"
    uid: 61163ea8-a60e-4de5-9dd3-667d3e4c62f7
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:47Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18304"
    uid: 107c6c52-6fa9-4a31-a7fd-e47f70095ee0
  kind: Event
  lastTimestamp: "2026-09-17T21:08:02Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-primary-b61317-predictor.18d637c1e4613602
    namespace: kserve-ci-e2e-test
    resourceVersion: "19266"
    uid: dc339519-bf18-4246-b11a-e2cf84dc507c
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:17Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "18133"
    uid: 5658b175-d82e-4483-b3ab-4d3779cf8c74
  kind: Event
  lastTimestamp: "2026-09-17T21:07:41Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:41Z"
    name: isvc-primary-b61317.18d637bae4c16954
    namespace: kserve-ci-e2e-test
    resourceVersion: "19084"
    uid: 65123afa-f022-4fd2-945b-fa8eb740294f
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:17Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "18133"
    uid: 5658b175-d82e-4483-b3ab-4d3779cf8c74
  kind: Event
  lastTimestamp: "2026-09-17T21:06:17Z"
  message: 'Failed to update status for InferenceService "isvc-primary-b61317": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-b61317":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:17Z"
    name: isvc-primary-b61317.18d637bae55a97de
    namespace: kserve-ci-e2e-test
    resourceVersion: "18148"
    uid: d7ea86e1-0636-4612-bc0d-ae008fddd48e
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:17Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "18133"
    uid: 5658b175-d82e-4483-b3ab-4d3779cf8c74
  kind: Event
  lastTimestamp: "2026-09-17T21:06:17Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-primary-b61317": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:06:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:17Z"
    name: isvc-primary-b61317.18d637bae55afa1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18149"
    uid: 7f5c9205-7e80-4a78-ba0a-d44ea4d48fe0
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "19082"
    uid: 5658b175-d82e-4483-b3ab-4d3779cf8c74
  kind: Event
  lastTimestamp: "2026-09-17T21:07:41Z"
  message: InferenceService [isvc-primary-b61317] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:07:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:41Z"
    name: isvc-primary-b61317.18d637ce606e518f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19083"
    uid: 9a1ea472-8bc5-4fec-b3e1-519012d9735a
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:01:07.569167Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14957"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63772ab7beb1d
    namespace: kserve-ci-e2e-test
    resourceVersion: "14964"
    uid: 113c58cd-55aa-4289-8611-e6c5712440d3
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14976"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Add eth0 [10.133.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63772cd73216a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15001"
    uid: 34ed223c-7bb6-455b-92b2-e93c99eeff0c
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63772cfa84c7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15010"
    uid: 55e82b0e-7427-4d99-847d-d95ffe8f3704
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63772d2c1f33b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15013"
    uid: 244c28df-520d-4e16-b203-efebb5eb8cbd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:08Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:01:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:08Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63772d3371bef
    namespace: kserve-ci-e2e-test
    resourceVersion: "15014"
    uid: 6be2d717-949e-44d9-9952-b2ba1bb961ad
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:12Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
  metadata:
    creationTimestamp: "2026-09-17T21:01:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:12Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63773bbe8dd54
    namespace: kserve-ci-e2e-test
    resourceVersion: "15048"
    uid: 7a83ecbf-8ad2-413c-8254-d0f87f6991bf
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    in 19.054s (19.054s including waiting). Image size: 1604543374 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d637782ba9bcb9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15206"
    uid: b0b2215a-35d8-4886-9a2d-0e45e2eb19a0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d637783058396b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15209"
    uid: 02ec8273-bb3c-40fd-a3f7-352240fedfe9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6377830f8b8ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "15213"
    uid: 522ed9dc-f064-4fc4-8b62-ea2608b6d0b2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:31Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T21:01:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:31Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d637783104bc2e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15214"
    uid: f9047068-cd01-47a3-8a6e-41e0a76d5fa3
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.885s (2.885s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63778dcfe0cdd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15245"
    uid: e193bc7b-6a64-40fb-8794-55a07b53a125
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63778e1395f24
    namespace: kserve-ci-e2e-test
    resourceVersion: "15248"
    uid: 4632132c-ce1f-45c1-a808-f2dbcf96b18c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63778e1beefff
    namespace: kserve-ci-e2e-test
    resourceVersion: "15250"
    uid: 9bd656ec-cad7-4d40-a747-642247ddd426
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:34Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:b59b759dbc00d731ff1eeb0e6abaa9d6adb19d1c613e234f3ace5c825641711a"
  metadata:
    creationTimestamp: "2026-09-17T21:01:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:34Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63778e1cd1cac
    namespace: kserve-ci-e2e-test
    resourceVersion: "15252"
    uid: 133845ac-74ea-46ea-9c9f-1ba9d57f4ea0
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:36Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:b59b759dbc00d731ff1eeb0e6abaa9d6adb19d1c613e234f3ace5c825641711a"
    in 2.685s (2.685s including waiting). Image size: 240300107 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:01:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:36Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6377981d9f747
    namespace: kserve-ci-e2e-test
    resourceVersion: "15279"
    uid: b3098fdb-f0b9-4ec6-b3f2-9cf0ad82f20b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:37Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-17T21:01:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:37Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63779867ec3d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15280"
    uid: d2c7e5be-f3bb-45e7-8ada-a83227663dde
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:01:37Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-17T21:01:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:01:37Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6377987c07567
    namespace: kserve-ci-e2e-test
    resourceVersion: "15281"
    uid: 27cd2819-4d9c-4304-b3da-91eeb9703fac
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:03:20Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:01:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:20Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d637799ab5407b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16300"
    uid: 85a69ec1-ee00-45a7-98ea-09a06ae2fb6d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:03:20Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T21:01:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:20Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d63779d684ca25
    namespace: kserve-ci-e2e-test
    resourceVersion: "16301"
    uid: 9664f06b-65bf-488b-8562-283f810f0e39
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6378fd0aef225
    namespace: kserve-ci-e2e-test
    resourceVersion: "16126"
    uid: 748a0e5b-8d7b-4b5d-843b-0de0df1c840f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6378fd0af2b60
    namespace: kserve-ci-e2e-test
    resourceVersion: "16129"
    uid: 6acd66c6-990b-48fc-80d4-320bdf386335
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6378fd0af5602
    namespace: kserve-ci-e2e-test
    resourceVersion: "16128"
    uid: ab434610-41ef-4156-9218-babb7f7b1a04
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 6a308e40-a9f1-4a70-aae9-eca1b7b30d37
  kind: Event
  lastTimestamp: "2026-09-17T21:03:19Z"
  message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp
    10.133.0.27:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:03:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:19Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl.18d6379030beef57
    namespace: kserve-ci-e2e-test
    resourceVersion: "16292"
    uid: 37fa742a-a9a8-4169-bd3f-4fe9f2b9c413
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb
    namespace: kserve-ci-e2e-test
    resourceVersion: "14945"
    uid: e3cfc0ef-d31d-4e09-a06c-f8251900bdfa
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: 'Created pod: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb-pslrl'
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb.18d63772ab174245
    namespace: kserve-ci-e2e-test
    resourceVersion: "14961"
    uid: ee3dd642-ec19-4f09-bcb8-fd4742896fe6
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-raw-sklearn-batcher-89dfa-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14941"
    uid: 8c2e2d14-356b-4108-bdd8-d0beaee22acb
  kind: Event
  lastTimestamp: "2026-09-17T21:01:07Z"
  message: Scaled up replica set isvc-raw-sklearn-batcher-89dfa-predictor-76d8855cfb
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:01:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor.18d63772a737e596
    namespace: kserve-ci-e2e-test
    resourceVersion: "14948"
    uid: d5036bf7-d9e7-43f0-9b94-6eb3a2af44ef
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:22Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-89dfa-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14944"
    uid: 09b05174-f602-4938-afbc-78ab0f5b2048
  kind: Event
  lastTimestamp: "2026-09-17T21:02:07Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:01:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:02:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor.18d6377626aa57c9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15523"
    uid: 2c0fd6a6-0bb9-4d0d-a08d-58b0186b811b
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:22Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-89dfa-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14944"
    uid: 09b05174-f602-4938-afbc-78ab0f5b2048
  kind: Event
  lastTimestamp: "2026-09-17T21:02:07Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:01:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:02:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor.18d637762716f746
    namespace: kserve-ci-e2e-test
    resourceVersion: "15524"
    uid: defcd793-3010-47d7-91cc-7f3da0e489eb
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:02:22Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-89dfa-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15139"
    uid: 09b05174-f602-4938-afbc-78ab0f5b2048
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:02:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor.18d6378423ca0052
    namespace: kserve-ci-e2e-test
    resourceVersion: "16061"
    uid: 1014d4fd-76f9-4828-b497-8ba3aa11fa97
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:02:22Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-89dfa-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15139"
    uid: 09b05174-f602-4938-afbc-78ab0f5b2048
  kind: Event
  lastTimestamp: "2026-09-17T21:03:07Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:02:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:07Z"
    name: isvc-raw-sklearn-batcher-89dfa-predictor.18d63784243b17ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "16062"
    uid: 6bbb3723-33f3-4f00-9c36-e5a63a90464e
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:01:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-89dfa
    namespace: kserve-ci-e2e-test
    resourceVersion: "14930"
    uid: 75dc8bcc-9a54-459b-a16e-7bc7cc2f60bf
  kind: Event
  lastTimestamp: "2026-09-17T21:03:00Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:01:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:00Z"
    name: isvc-raw-sklearn-batcher-89dfa.18d63772a73b1d44
    namespace: kserve-ci-e2e-test
    resourceVersion: "15962"
    uid: a5faaa09-d13e-4b3f-a880-cd276e31c744
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-89dfa
    namespace: kserve-ci-e2e-test
    resourceVersion: "15958"
    uid: 75dc8bcc-9a54-459b-a16e-7bc7cc2f60bf
  kind: Event
  lastTimestamp: "2026-09-17T21:03:00Z"
  message: InferenceService [isvc-raw-sklearn-batcher-89dfa] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:03:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:00Z"
    name: isvc-raw-sklearn-batcher-89dfa.18d6378cf7f07b3b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15959"
    uid: 7a627678-db7d-4792-8dad-59c11bb5a51e
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:08:02.986007Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19271"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:08:02Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d36441ea89
    namespace: kserve-ci-e2e-test
    resourceVersion: "19277"
    uid: 6e31f043-a4ef-4954-90d9-6b9be5a9116c
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19275"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:03Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-b61317-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:08:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:03Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d3708f90e7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19286"
    uid: 9147a969-9457-45bd-8ae5-84f7f83865a6
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:04Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19281"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:04Z"
  message: Add eth0 [10.133.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:08:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:08:04Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d3a9ec8c8b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19301"
    uid: 34c7bd39-87f0-4465-b8c7-4526d3549b6e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19275"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:08Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:08Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d3ab554f61
    namespace: kserve-ci-e2e-test
    resourceVersion: "19338"
    uid: fd25bd83-30dc-4288-902a-e937d587a4b0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19275"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:08Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:08:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:08Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d3ae72637e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19340"
    uid: 5994591d-cba7-46ce-a965-b3fa4b90ecf5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19275"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:08Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:08:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:08Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d3aee15535
    namespace: kserve-ci-e2e-test
    resourceVersion: "19341"
    uid: 7e7755a0-0aa5-4e1f-a967-501c5b734790
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19275"
    uid: b5c7b2df-6d85-45f5-9190-d81cf88321e6
  kind: Event
  lastTimestamp: "2026-09-17T21:08:12Z"
  message: Back-off restarting failed container storage-initializer in pod isvc-secondary-b61317-predictor-6475c545c4-dk2kz_kserve-ci-e2e-test(b5c7b2df-6d85-45f5-9190-d81cf88321e6)
  metadata:
    creationTimestamp: "2026-09-17T21:08:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:12Z"
    name: isvc-secondary-b61317-predictor-6475c545c4-dk2kz.18d637d580f99a68
    namespace: kserve-ci-e2e-test
    resourceVersion: "19372"
    uid: ddf0e4a0-bc8e-4f19-8819-4aa1934b25fa
  reason: BackOff
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-secondary-b61317-predictor-6475c545c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19260"
    uid: 2f75a63d-09e5-4e99-83cd-2a2dc004ff68
  kind: Event
  lastTimestamp: "2026-09-17T21:08:02Z"
  message: 'Created pod: isvc-secondary-b61317-predictor-6475c545c4-dk2kz'
  metadata:
    creationTimestamp: "2026-09-17T21:08:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-secondary-b61317-predictor-6475c545c4.18d637d363d8bb01
    namespace: kserve-ci-e2e-test
    resourceVersion: "19274"
    uid: e3fdd19e-a725-473e-afd0-c314ffc291e5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-secondary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19258"
    uid: 3078c648-95f0-47c6-adf8-082d8efe9295
  kind: Event
  lastTimestamp: "2026-09-17T21:08:02Z"
  message: Scaled up replica set isvc-secondary-b61317-predictor-6475c545c4 from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T21:08:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-secondary-b61317-predictor.18d637d360cfd373
    namespace: kserve-ci-e2e-test
    resourceVersion: "19263"
    uid: 46cacf2d-f70a-44b8-95e6-d319b4b51bdd
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:17Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19259"
    uid: cd0c9fdd-68b7-48f8-b341-8d523c15ca9d
  kind: Event
  lastTimestamp: "2026-09-17T21:08:17Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:08:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:17Z"
    name: isvc-secondary-b61317-predictor.18d637d6dfb52372
    namespace: kserve-ci-e2e-test
    resourceVersion: "19417"
    uid: cf3855a2-c3d5-460d-bfed-30a9b5c98061
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:17Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-b61317-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19259"
    uid: cd0c9fdd-68b7-48f8-b341-8d523c15ca9d
  kind: Event
  lastTimestamp: "2026-09-17T21:08:17Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:08:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:17Z"
    name: isvc-secondary-b61317-predictor.18d637d6e02069ee
    namespace: kserve-ci-e2e-test
    resourceVersion: "19420"
    uid: d9cb58df-4818-4200-ad30-97700b9792b3
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "19251"
    uid: 9c6587f2-5f5a-4bcf-b439-097962636b95
  kind: Event
  lastTimestamp: "2026-09-17T21:08:12Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:08:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:12Z"
    name: isvc-secondary-b61317.18d637d360d13a1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19377"
    uid: b71dc25d-d007-48ea-9ee9-13af38f4a5c1
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "19251"
    uid: 9c6587f2-5f5a-4bcf-b439-097962636b95
  kind: Event
  lastTimestamp: "2026-09-17T21:08:02Z"
  message: 'Failed to update status for InferenceService "isvc-secondary-b61317":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b61317":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-secondary-b61317.18d637d36157a633
    namespace: kserve-ci-e2e-test
    resourceVersion: "19265"
    uid: 6567ef0d-aa8d-439f-bb5c-5f925b0bea28
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-b61317
    namespace: kserve-ci-e2e-test
    resourceVersion: "19251"
    uid: 9c6587f2-5f5a-4bcf-b439-097962636b95
  kind: Event
  lastTimestamp: "2026-09-17T21:08:02Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-secondary-b61317": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:02Z"
    name: isvc-secondary-b61317.18d637d36157f83d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19268"
    uid: 7d71baa1-794b-4f44-b756-ac77c5f2d94b
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:03:12.885269Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16154"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d6378fd8e7e444
    namespace: kserve-ci-e2e-test
    resourceVersion: "16160"
    uid: 702072ca-dd01-4e49-97cc-cdd13eefa530
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16166"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: Add eth0 [10.132.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d6378ffaee7c1a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16205"
    uid: 863dbda0-90df-4850-9777-5a90856170d2
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d6378ffc712b37
    namespace: kserve-ci-e2e-test
    resourceVersion: "16207"
    uid: 0b83abd5-2355-4bda-bc9d-8adb57a86259
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d6378fffba075f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16208"
    uid: 4fd16b65-237d-4c6b-89b2-75b9427b803e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790002e76a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16209"
    uid: 662eb6ed-85d6-439e-a320-b725c61920c2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790eb3a81e0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16262"
    uid: 12eba5af-8c37-4d2c-9c7f-ce30e22a4b05
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790ee5f795c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16266"
    uid: fcf9db5b-bbb1-4f2c-af3c-c26251c3bf16
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790eed79382
    namespace: kserve-ci-e2e-test
    resourceVersion: "16267"
    uid: 301d63b6-3612-480d-a622-269745a0e24d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790eee30899
    namespace: kserve-ci-e2e-test
    resourceVersion: "16268"
    uid: daf97572-9419-4936-b180-b9e8d70401a7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790f274f789
    namespace: kserve-ci-e2e-test
    resourceVersion: "16273"
    uid: 73108464-e6e3-42f7-b295-2c3c7f101652
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d63790f2eb3f00
    namespace: kserve-ci-e2e-test
    resourceVersion: "16274"
    uid: 697d5d5f-245a-481c-a5b2-2c06f0f3e243
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:04:32Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:03:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:32Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d637916348b3d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16977"
    uid: a0c2557e-7864-4126-994a-8b83beb23814
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d637a98094f41f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17346"
    uid: f371b9ca-795e-4aa2-9c84-8eb011ef4724
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d637a98096156c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17349"
    uid: d11b5f3f-56e3-4888-9e87-af87b3ed4e2b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h
    namespace: kserve-ci-e2e-test
    resourceVersion: "16156"
    uid: e5f88102-8b3a-48d6-9f09-96aae9f59be9
  kind: Event
  lastTimestamp: "2026-09-17T21:05:05Z"
  message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp
    10.132.0.26:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:05:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:05Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h.18d637aa119a2be4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17465"
    uid: c08f065a-3e0b-45e1-b32d-1baa254de3f4
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468
    namespace: kserve-ci-e2e-test
    resourceVersion: "16145"
    uid: aafabea5-c4a5-412f-b266-5c143488f368
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: 'Created pod: isvc-sklearn-graph-raw-669c3-predictor-b5f549468-wfk6h'
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-sklearn-graph-raw-669c3-predictor-b5f549468.18d6378fd8781f04
    namespace: kserve-ci-e2e-test
    resourceVersion: "16155"
    uid: a1703730-5530-4774-be84-f64088bca088
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: b0fc70fc-c7cd-4e94-8e36-6edd195e1c03
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-669c3-predictor-b5f549468
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-sklearn-graph-raw-669c3-predictor.18d6378fd55604ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "16146"
    uid: 7f71b627-f634-4c47-9d56-1de1de6fe824
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16144"
    uid: 4f99a936-852f-4033-b278-632459ab19db
  kind: Event
  lastTimestamp: "2026-09-17T21:03:42Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:03:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:42Z"
    name: isvc-sklearn-graph-raw-669c3-predictor.18d6379354a30f5d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16477"
    uid: 467f25c8-ae43-44d6-9eb8-8b44e1d2c82c
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16144"
    uid: 4f99a936-852f-4033-b278-632459ab19db
  kind: Event
  lastTimestamp: "2026-09-17T21:03:42Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:03:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:42Z"
    name: isvc-sklearn-graph-raw-669c3-predictor.18d63793552f2592
    namespace: kserve-ci-e2e-test
    resourceVersion: "16478"
    uid: a73cf8f5-3641-4fa5-bdcf-ca6856cf5e6a
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16358"
    uid: 4f99a936-852f-4033-b278-632459ab19db
  kind: Event
  lastTimestamp: "2026-09-17T21:04:57Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:03:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:57Z"
    name: isvc-sklearn-graph-raw-669c3-predictor.18d6379a538346f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17254"
    uid: 03a4290e-6afa-470c-b801-b5131d48db77
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16358"
    uid: 4f99a936-852f-4033-b278-632459ab19db
  kind: Event
  lastTimestamp: "2026-09-17T21:04:57Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:03:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:57Z"
    name: isvc-sklearn-graph-raw-669c3-predictor.18d6379a545bd2e7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17255"
    uid: d5bc2e6a-9eea-4a57-8d14-d266db0fce7a
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16131"
    uid: a95a00cd-e37c-4ee7-bf65-cba738120cae
  kind: Event
  lastTimestamp: "2026-09-17T21:04:42Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:42Z"
    name: isvc-sklearn-graph-raw-669c3.18d6378fd58f5ac2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17099"
    uid: 9bbfabec-75c9-4e12-b7b8-420004fce541
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16131"
    uid: a95a00cd-e37c-4ee7-bf65-cba738120cae
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-669c3":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-669c3":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-sklearn-graph-raw-669c3.18d6378fd660c256
    namespace: kserve-ci-e2e-test
    resourceVersion: "16150"
    uid: b6a19245-2c1e-45f5-82d0-bb57f97c7bf9
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17096"
    uid: a95a00cd-e37c-4ee7-bf65-cba738120cae
  kind: Event
  lastTimestamp: "2026-09-17T21:04:42Z"
  message: InferenceService [isvc-sklearn-graph-raw-669c3] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:04:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:42Z"
    name: isvc-sklearn-graph-raw-669c3.18d637a4a8bf1008
    namespace: kserve-ci-e2e-test
    resourceVersion: "17097"
    uid: 3c9b7024-2f02-427b-ba58-b97b9b0cc5fd
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:05:03.229040Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17382"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637a989e83414
    namespace: kserve-ci-e2e-test
    resourceVersion: "17388"
    uid: 789336f4-9a6b-4b93-9555-9287a7777329
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17402"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Add eth0 [10.133.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637a9abf7c14d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17433"
    uid: 035c2d26-5422-4074-b939-4cd9927532e9
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637a9ad42dffd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17435"
    uid: 6b06daa6-c646-42f8-bdc6-51df69a66daf
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637a9afee39ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "17436"
    uid: 8d178153-924a-41dc-932e-c0e5e29a6c70
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637a9b05ce3d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17437"
    uid: 25d05304-2bbe-4c30-bdf5-c4ce42118ef4
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637aab50200c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17490"
    uid: 35181371-4921-4484-a28f-01460725e750
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637aab87bdd6c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17493"
    uid: 266fb2bb-5f38-47f4-a71c-61075cf851b5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637aab8ed58d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17496"
    uid: 98b0d6e2-4ac8-4688-9a79-5680bf960487
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637aab8f6e861
    namespace: kserve-ci-e2e-test
    resourceVersion: "17497"
    uid: b2a683cd-c7af-41ae-adc2-f9c913f1dc01
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637aabc0d2dcc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17501"
    uid: 484bbd4b-95ca-4d03-adb3-0037fe10f3ae
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637aabc79eb5e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17502"
    uid: 2cdcbe0e-695f-4b7f-b469-f75ab759434e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:05:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637ab2d1e8c02
    namespace: kserve-ci-e2e-test
    resourceVersion: "18644"
    uid: 81d964fe-9344-48b1-b120-bfe4a4590905
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637c58757ee38
    namespace: kserve-ci-e2e-test
    resourceVersion: "18635"
    uid: 232c04f9-94b7-454a-896b-481ee14abea4
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637c58758cc74
    namespace: kserve-ci-e2e-test
    resourceVersion: "18638"
    uid: 4f161548-36bd-49f7-b915-cb0ebf36c2d1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm
    namespace: kserve-ci-e2e-test
    resourceVersion: "17385"
    uid: 199305ac-35b9-4ccc-9ef4-7bb783c42118
  kind: Event
  lastTimestamp: "2026-09-17T21:07:06Z"
  message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp
    10.133.0.31:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:07:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:06Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm.18d637c62f86ab94
    namespace: kserve-ci-e2e-test
    resourceVersion: "18719"
    uid: 6db08df3-49f5-4d0e-8e84-7217b8eb0b0b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56
    namespace: kserve-ci-e2e-test
    resourceVersion: "17371"
    uid: c2dcc88d-cd94-4965-9b9b-6733b2ab6aa5
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: 'Created pod: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56-bmzxm'
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56.18d637a98974c8ee
    namespace: kserve-ci-e2e-test
    resourceVersion: "17384"
    uid: 4349018f-834a-4f63-9154-497d334ec419
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17369"
    uid: d5adb47a-b468-43b0-bf05-6e23a512784f
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-hpa-e3e0d-predictor-d6fc66f56
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor.18d637a9862d7ae2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17374"
    uid: 9963405b-d0ba-4711-b728-46222514931f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17370"
    uid: 77de6ee8-20bc-4ffd-b8b2-7cdb012962ce
  kind: Event
  lastTimestamp: "2026-09-17T21:05:33Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:05:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:05:33Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor.18d637ad0632b151
    namespace: kserve-ci-e2e-test
    resourceVersion: "17712"
    uid: fab7c8b4-b8e2-453d-9097-98de17026d2d
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17370"
    uid: 77de6ee8-20bc-4ffd-b8b2-7cdb012962ce
  kind: Event
  lastTimestamp: "2026-09-17T21:05:33Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:05:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:05:33Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor.18d637ad06e24099
    namespace: kserve-ci-e2e-test
    resourceVersion: "17713"
    uid: 999dd430-5acb-493a-a570-07669ef26c40
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:48Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17584"
    uid: 77de6ee8-20bc-4ffd-b8b2-7cdb012962ce
  kind: Event
  lastTimestamp: "2026-09-17T21:06:48Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:05:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:48Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor.18d637b404c72fd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18437"
    uid: 50d779fc-73aa-4b5d-ac4c-2deaca63758b
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:48Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17584"
    uid: 77de6ee8-20bc-4ffd-b8b2-7cdb012962ce
  kind: Event
  lastTimestamp: "2026-09-17T21:06:48Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:05:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:48Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d-predictor.18d637b4053354d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "18438"
    uid: 5f01c8cf-b3e3-4d2c-87c8-653b48c68437
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17356"
    uid: ca9657b9-eb75-4839-924e-5268743c9acf
  kind: Event
  lastTimestamp: "2026-09-17T21:06:33Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:33Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d.18d637a9862d4bfc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: de90c86c-1877-40da-b6a6-cfbb225df195
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:33Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18320"
    uid: ca9657b9-eb75-4839-924e-5268743c9acf
  kind: Event
  lastTimestamp: "2026-09-17T21:06:33Z"
  message: InferenceService [isvc-sklearn-graph-raw-hpa-e3e0d] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:06:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:33Z"
    name: isvc-sklearn-graph-raw-hpa-e3e0d.18d637be934294f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18322"
    uid: d514a452-6ed2-429f-8aee-40b224028e8d
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:08:49.286223Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19892"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637de2bf7056f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19897"
    uid: 3eb0ca75-c9e7-4be6-9ed2-7fee7fd1f4df
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-f9ecc-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637de3597a524
    namespace: kserve-ci-e2e-test
    resourceVersion: "19908"
    uid: c5531dd9-84b5-4305-a095-04effa0bb872
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:50Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19903"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:50Z"
  message: Add eth0 [10.132.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:08:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:08:50Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637de71c23896
    namespace: kserve-ci-e2e-test
    resourceVersion: "19922"
    uid: 38e8777e-1171-4c91-ae58-04921762f007
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:50Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:50Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637de733adf63
    namespace: kserve-ci-e2e-test
    resourceVersion: "19924"
    uid: ff5bf0d5-afb6-45ff-b5e6-fd054e201b93
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:50Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:08:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:50Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637de765ff43c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19925"
    uid: 3738f1c2-935b-43c0-8101-e5dc88197490
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:50Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:08:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:50Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637de76d275f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19926"
    uid: e3a3c74b-4bba-4d6e-93f6-c93e54f247b3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:54Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:54Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637df7fc64c69
    namespace: kserve-ci-e2e-test
    resourceVersion: "19972"
    uid: 3a3a0e81-7fac-406f-9407-313a6216dc89
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:55Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:08:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:55Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637df832236ce
    namespace: kserve-ci-e2e-test
    resourceVersion: "19976"
    uid: b86f9e70-abd4-46de-a181-ffe90eab31f1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:55Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:08:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:55Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637df8394dcea
    namespace: kserve-ci-e2e-test
    resourceVersion: "19977"
    uid: 549d66b4-8132-4765-8dde-9e915bb757a3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:55Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:55Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637df83a14f83
    namespace: kserve-ci-e2e-test
    resourceVersion: "19978"
    uid: 2b53a1a5-8764-4734-9865-d07d125159ba
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:55Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:08:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:55Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637df867c8462
    namespace: kserve-ci-e2e-test
    resourceVersion: "19980"
    uid: 6e6aa8d7-9654-4417-be4f-3884d51d0ae7
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:55Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:08:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:55Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637df86edd18d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19982"
    uid: 66e24cb9-a476-43d3-ac7d-b09f8d82142d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 20
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:08:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d637dfbbd50917
    namespace: kserve-ci-e2e-test
    resourceVersion: "21448"
    uid: 3ca89b58-12bd-46cf-981f-7952b8f4b578
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:11:40Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:40Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d63806060c9f8c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21325"
    uid: 0829bc6e-8c74-4040-a54c-3e4dac982a58
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:11:40Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:40Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d63806060d3b19
    namespace: kserve-ci-e2e-test
    resourceVersion: "21326"
    uid: 381f2061-4964-4779-9c20-9669d984e0bf
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19894"
    uid: 3ad76ac7-1b45-4a8b-9472-a3902021e09f
  kind: Event
  lastTimestamp: "2026-09-17T21:11:51Z"
  message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp
    10.132.0.31:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:11:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:52Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7.18d63806625b6e7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21553"
    uid: 77038956-0a40-4056-bccb-62e993317396
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54
    namespace: kserve-ci-e2e-test
    resourceVersion: "19883"
    uid: 03e3d1be-eb8d-4f68-88f1-a6cdca91cc47
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: 'Created pod: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54-zx8s7'
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54.18d637de2b812f98
    namespace: kserve-ci-e2e-test
    resourceVersion: "19893"
    uid: 6818d0c7-c705-4a11-96b6-5e00c7e8c550
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-scale-raw-f9ecc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19881"
    uid: 420251f8-2de9-4108-8f40-4656d8a7210e
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: Scaled up replica set isvc-sklearn-scale-raw-f9ecc-predictor-5df6567f54
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor.18d637de290788c7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19885"
    uid: 3ca70210-f345-4266-bc0a-3c2f6988135c
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:09:04Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-f9ecc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19884"
    uid: 438476cc-ca1e-4a69-8cbd-1af8b4bc6322
  kind: Event
  lastTimestamp: "2026-09-17T21:09:34Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:09:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:09:34Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor.18d637e1a7d0fe9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20282"
    uid: e2c3c2c9-79ea-45b9-96d2-d465249e08cb
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:09:04Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-f9ecc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19884"
    uid: 438476cc-ca1e-4a69-8cbd-1af8b4bc6322
  kind: Event
  lastTimestamp: "2026-09-17T21:09:34Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:09:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:09:34Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor.18d637e1a8487e59
    namespace: kserve-ci-e2e-test
    resourceVersion: "20283"
    uid: 0f745436-c498-46f6-b9e4-20677439fb60
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:09:19Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-f9ecc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20062"
    uid: 438476cc-ca1e-4a69-8cbd-1af8b4bc6322
  kind: Event
  lastTimestamp: "2026-09-17T21:11:34Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:09:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:34Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor.18d637e5278a711c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21248"
    uid: 0c4df5a3-c187-43ac-ba82-1468f895a527
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:09:19Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-f9ecc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20062"
    uid: 438476cc-ca1e-4a69-8cbd-1af8b4bc6322
  kind: Event
  lastTimestamp: "2026-09-17T21:11:34Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:09:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:34Z"
    name: isvc-sklearn-scale-raw-f9ecc-predictor.18d637e527fb0b24
    namespace: kserve-ci-e2e-test
    resourceVersion: "21249"
    uid: a52dddb3-1fb0-4374-9383-eb9489892562
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-f9ecc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19869"
    uid: 0d33b5e4-3cc6-4e89-b0f6-36d4a35842d5
  kind: Event
  lastTimestamp: "2026-09-17T21:11:36Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:11:36Z"
    name: isvc-sklearn-scale-raw-f9ecc.18d637de2930ea9b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21269"
    uid: 8687af5d-910b-4c01-a492-d165ac0fa4df
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-f9ecc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19869"
    uid: 0d33b5e4-3cc6-4e89-b0f6-36d4a35842d5
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-f9ecc":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-f9ecc":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-sklearn-scale-raw-f9ecc.18d637de2c2cc5a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19898"
    uid: 7e2311de-84f3-4912-b523-45aedf84ef47
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-f9ecc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19869"
    uid: 0d33b5e4-3cc6-4e89-b0f6-36d4a35842d5
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-f9ecc": the object
    has been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: isvc-sklearn-scale-raw-f9ecc.18d637de2c2d36b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "19901"
    uid: c8e242f7-574d-43d8-b187-974281388ceb
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-f9ecc
    namespace: kserve-ci-e2e-test
    resourceVersion: "21267"
    uid: 0d33b5e4-3cc6-4e89-b0f6-36d4a35842d5
  kind: Event
  lastTimestamp: "2026-09-17T21:11:36Z"
  message: InferenceService [isvc-sklearn-scale-raw-f9ecc] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:11:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:11:36Z"
    name: isvc-sklearn-scale-raw-f9ecc.18d638050b97a4ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "21268"
    uid: fff3b39f-acb8-493f-9fa3-c41d12784235
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:03:13.026762Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16186"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d6378fe156eda8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16190"
    uid: d3b47afa-0c09-4da1-a809-5027f642ace5
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16195"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: Add eth0 [10.132.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637900407b270
    namespace: kserve-ci-e2e-test
    resourceVersion: "16212"
    uid: 320664a1-13c7-4e7b-b585-db6e7d9ee349
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d6379005897a78
    namespace: kserve-ci-e2e-test
    resourceVersion: "16214"
    uid: 8f770e0b-252a-44ef-9385-200efc255805
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d6379008c1414b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16215"
    uid: 9f18521e-0647-444e-b557-21dc33731366
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637900935e347
    namespace: kserve-ci-e2e-test
    resourceVersion: "16216"
    uid: 45873b26-fe3b-4ac2-abe8-0df72708f195
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:17Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-09-17T21:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:17Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d63790eb2a3968
    namespace: kserve-ci-e2e-test
    resourceVersion: "16261"
    uid: f0f13f6e-3a29-492e-8bec-dab3b312176d
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:35Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.413s (18.413s
    including waiting). Image size: 1435742980 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:03:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:35Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d6379534ab6783
    namespace: kserve-ci-e2e-test
    resourceVersion: "16418"
    uid: e2668ab3-757c-4a32-89ba-0f09ab4d9967
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:35Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:03:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:35Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637953865c550
    namespace: kserve-ci-e2e-test
    resourceVersion: "16419"
    uid: 7eee7d96-cfe8-4c19-b7d4-867403e1b47b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:35Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:03:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:35Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d6379538dbf4df
    namespace: kserve-ci-e2e-test
    resourceVersion: "16420"
    uid: 483f2726-f7bb-49d0-84de-45916d4f9afb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:35Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:03:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:35Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d6379538e78de0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16422"
    uid: dd6acf1a-487b-43d9-95df-ce8e87b4a641
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:36Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:03:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:36Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637953bfa4237
    namespace: kserve-ci-e2e-test
    resourceVersion: "16423"
    uid: dadd97d8-80ff-4573-9670-40a683c0325a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:03:36Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:03:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:03:36Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637953c6d4a42
    namespace: kserve-ci-e2e-test
    resourceVersion: "16424"
    uid: f484e7b4-3b7d-4039-97f4-56bb89361686
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:04:42Z"
  message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:03:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:42Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637955e4c5fcf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17101"
    uid: 34b6d8bc-3af1-4d7a-9528-75d68e2a1739
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637a9830eb83a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17361"
    uid: 436283c7-4e9a-48bd-b0f8-66186a38a8de
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16188"
    uid: a7079774-6102-4c26-b948-55bdaf0bd825
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns.18d637a9830f09b6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17362"
    uid: d263ce75-b085-4bb7-953f-5f8c822e67fc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:13Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85
    namespace: kserve-ci-e2e-test
    resourceVersion: "16176"
    uid: 802bce72-eece-4ceb-ac61-f7331d02256c
  kind: Event
  lastTimestamp: "2026-09-17T21:03:13Z"
  message: 'Created pod: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85-7lqns'
  metadata:
    creationTimestamp: "2026-09-17T21:03:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor-59fd788c85.18d6378fe0b1aab0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16187"
    uid: f06bd5de-55d2-4f80-ab3f-0ff8c12c1948
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16173"
    uid: 7f223351-b747-4ecf-b2a1-987be1b51806
  kind: Event
  lastTimestamp: "2026-09-17T21:03:12Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-669c3-predictor-59fd788c85
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:03:12Z"
    name: isvc-xgboost-graph-raw-669c3-predictor.18d6378fde028068
    namespace: kserve-ci-e2e-test
    resourceVersion: "16178"
    uid: 518ee2c7-3d39-4705-9e11-fe3070f942e8
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16175"
    uid: 1cf78d24-f5f2-4839-9ec4-799e69db70e1
  kind: Event
  lastTimestamp: "2026-09-17T21:04:13Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:03:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor.18d637935d07bbfe
    namespace: kserve-ci-e2e-test
    resourceVersion: "16708"
    uid: fb247f01-c402-4ed9-970b-b30b116b8ad4
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:28Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16175"
    uid: 1cf78d24-f5f2-4839-9ec4-799e69db70e1
  kind: Event
  lastTimestamp: "2026-09-17T21:04:13Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:03:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:13Z"
    name: isvc-xgboost-graph-raw-669c3-predictor.18d637935df20b47
    namespace: kserve-ci-e2e-test
    resourceVersion: "16709"
    uid: 6ea40ac1-048e-40ca-b223-4c912a1cf5af
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16364"
    uid: 1cf78d24-f5f2-4839-9ec4-799e69db70e1
  kind: Event
  lastTimestamp: "2026-09-17T21:04:58Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:58Z"
    name: isvc-xgboost-graph-raw-669c3-predictor.18d637a15aa67193
    namespace: kserve-ci-e2e-test
    resourceVersion: "17256"
    uid: 74275bec-fe72-46e8-9172-355147d58153
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:28Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-669c3-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16364"
    uid: 1cf78d24-f5f2-4839-9ec4-799e69db70e1
  kind: Event
  lastTimestamp: "2026-09-17T21:04:58Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:04:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:58Z"
    name: isvc-xgboost-graph-raw-669c3-predictor.18d637a15bc5868e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17257"
    uid: c0939cae-f641-4d1d-b315-63631d29241e
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:03:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16167"
    uid: 24b3db01-c0cd-4ee0-82fb-aac013318690
  kind: Event
  lastTimestamp: "2026-09-17T21:04:52Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:52Z"
    name: isvc-xgboost-graph-raw-669c3.18d6378fde016404
    namespace: kserve-ci-e2e-test
    resourceVersion: "17182"
    uid: 0b784e8e-59bc-4fd8-b6bd-0e26b7496c62
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:52Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17179"
    uid: 24b3db01-c0cd-4ee0-82fb-aac013318690
  kind: Event
  lastTimestamp: "2026-09-17T21:04:52Z"
  message: InferenceService [isvc-xgboost-graph-raw-669c3] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:04:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:52Z"
    name: isvc-xgboost-graph-raw-669c3.18d637a71609a86e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17180"
    uid: e9168ec9-7d3a-4c7d-99a4-5399203f1b7e
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:05:03.316650Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17411"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637a98f2108ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "17415"
    uid: 51b8d0a9-17f1-4be0-89ae-3267b4b3e079
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-e3e0d-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637a99c1a8236
    namespace: kserve-ci-e2e-test
    resourceVersion: "17427"
    uid: 4d0ecbc7-040b-410c-a413-3e380bddd53c
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:04Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17421"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:04Z"
  message: Add eth0 [10.133.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:05:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:05:04Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637a9d4cf73a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17448"
    uid: b3748a50-4bbd-4803-bbd4-7324cb031cae
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:04Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:05:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:04Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637a9d6222a7e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17450"
    uid: 5e33e40c-a45a-4677-b046-bd4225cf4f15
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:04Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:05:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:04Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637a9d8f189fa
    namespace: kserve-ci-e2e-test
    resourceVersion: "17451"
    uid: 51cc5fc8-3ed1-470b-b819-64f76555cfbd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:04Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:05:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:04Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637a9d95e0491
    namespace: kserve-ci-e2e-test
    resourceVersion: "17452"
    uid: 305f0cf1-a12d-452d-95e5-7dc45303e9b7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:08Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-09-17T21:05:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:08Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aab4ef04d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: a9869e49-ff94-4b2f-9861-303b4413a76a
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:26Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.31s (18.31s
    including waiting). Image size: 1435742980 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:26Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aef852a37b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17650"
    uid: a4a3b0d6-a214-464f-b021-5cb03bf90e6d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:26Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:26Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aefbcea756
    namespace: kserve-ci-e2e-test
    resourceVersion: "17652"
    uid: e76b8721-7429-4bab-90a1-b537fb1bcc16
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:26Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:26Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aefc3e5801
    namespace: kserve-ci-e2e-test
    resourceVersion: "17654"
    uid: f7f35d56-2879-465c-9083-bddb9a993720
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:26Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:26Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aefc491efa
    namespace: kserve-ci-e2e-test
    resourceVersion: "17655"
    uid: c1dbd9ca-c8ea-47c2-a306-a316c20895d5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:26Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:26Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aeff61abdb
    namespace: kserve-ci-e2e-test
    resourceVersion: "17656"
    uid: 23315263-7546-4d6b-9806-4cd617046be9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:05:26Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:26Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637aeffcca3e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17657"
    uid: 4bbc0ef2-f81b-42a8-9c16-819ad8fe5d31
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:07:04Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:05:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:04Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637af63ab3db7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18697"
    uid: f6d379d7-3ecd-412e-97eb-1cb2e18a6d8a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637c58a5f1822
    namespace: kserve-ci-e2e-test
    resourceVersion: "18649"
    uid: 44bd43c8-b046-4874-a51f-4f43df48d782
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637c58a5f8a4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18650"
    uid: af6e1742-ddfa-4cdb-95e8-17dddb7aeee3
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17412"
    uid: 432c0a3e-23a6-45ba-81f4-7cae8de3f677
  kind: Event
  lastTimestamp: "2026-09-17T21:07:04Z"
  message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp
    10.133.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:07:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:04Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5.18d637c5bdeddf31
    namespace: kserve-ci-e2e-test
    resourceVersion: "18694"
    uid: 549e48e1-a634-4b36-9204-914b68ecedc7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669
    namespace: kserve-ci-e2e-test
    resourceVersion: "17401"
    uid: 771303e7-6706-4ae1-81ef-5e4a52756fbe
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: 'Created pod: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669-9ckl5'
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669.18d637a98ec73445
    namespace: kserve-ci-e2e-test
    resourceVersion: "17413"
    uid: bb99e693-1a79-402a-b04d-47c97dd35507
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17398"
    uid: 20edc659-2ba0-4c56-ae1d-99e0e6f77f50
  kind: Event
  lastTimestamp: "2026-09-17T21:05:03Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-hpa-e3e0d-predictor-677b464669
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:05:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor.18d637a98c1f1fc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17405"
    uid: 561e97df-14a2-4963-b0e7-a8263eb11ac3
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17400"
    uid: e8fbcbfa-a82a-42c2-ab7b-1d7c73aed792
  kind: Event
  lastTimestamp: "2026-09-17T21:06:03Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:05:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor.18d637ad0ac07a9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17961"
    uid: 9c67fa4e-241d-40da-ab51-b99f4f8590fe
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17400"
    uid: e8fbcbfa-a82a-42c2-ab7b-1d7c73aed792
  kind: Event
  lastTimestamp: "2026-09-17T21:06:03Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:05:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor.18d637ad0b934e45
    namespace: kserve-ci-e2e-test
    resourceVersion: "17962"
    uid: 833ad282-3158-4a80-995d-eb694a968023
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17587"
    uid: e8fbcbfa-a82a-42c2-ab7b-1d7c73aed792
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:06:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor.18d637bb06ca8b87
    namespace: kserve-ci-e2e-test
    resourceVersion: "18615"
    uid: 70e872f5-24d6-44ae-905a-34b62e628d58
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17587"
    uid: e8fbcbfa-a82a-42c2-ab7b-1d7c73aed792
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:06:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d-predictor.18d637bb0760ce68
    namespace: kserve-ci-e2e-test
    resourceVersion: "18619"
    uid: e319f247-27ae-46a3-ba21-a1e130416cd6
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17381"
    uid: e51ec995-e2c6-428a-975e-793e22ebd728
  kind: Event
  lastTimestamp: "2026-09-17T21:06:34Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:05:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:34Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d.18d637a98c202eb4
    namespace: kserve-ci-e2e-test
    resourceVersion: "18336"
    uid: e7aae0c1-ddf0-4390-bfd1-cdb159ddd072
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:34Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18334"
    uid: e51ec995-e2c6-428a-975e-793e22ebd728
  kind: Event
  lastTimestamp: "2026-09-17T21:06:34Z"
  message: InferenceService [isvc-xgboost-graph-raw-hpa-e3e0d] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:06:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:34Z"
    name: isvc-xgboost-graph-raw-hpa-e3e0d.18d637bec7b5d5ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "18335"
    uid: 37a75094-dc2d-4fc9-9680-f03aa5ca8ae4
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:07:03.595990Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18669"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c590564f5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18673"
    uid: 89a54f8a-ed96-43c5-b538-abd171fa35ec
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:04Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18677"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:04Z"
  message: Add eth0 [10.132.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:07:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:07:04Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5b2844e0f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18691"
    uid: 593c2663-ca11-4d75-a2aa-84027865b1d3
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:04Z"
  message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
  metadata:
    creationTimestamp: "2026-09-17T21:07:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:04Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5b3ff7540
    namespace: kserve-ci-e2e-test
    resourceVersion: "18693"
    uid: ed40b445-6942-44a6-9bb7-5ecdbfb3a727
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:05Z"
  message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    in 1.144s (1.144s including waiting). Image size: 14813193 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:05Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5f83edcd3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18701"
    uid: 566fbda9-276a-476b-a5b0-e40cf3733c86
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:05Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:05Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5fb93ed61
    namespace: kserve-ci-e2e-test
    resourceVersion: "18703"
    uid: 58d30a47-380f-486f-a25b-91d7754a4da6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:05Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:05Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5fc0a64b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "18704"
    uid: 5e1f21a8-8a3c-4917-94f8-72eb881135d2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:05Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:05Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5fc14eb15
    namespace: kserve-ci-e2e-test
    resourceVersion: "18705"
    uid: 6bca370e-fecc-49fd-aa52-ff1ee4798702
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:05Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:05Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c5ffdd7c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18706"
    uid: ad20ea7f-bf83-4112-a4b3-e7da15eaff8d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:07:05Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:05Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637c60051525d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18707"
    uid: 31f95a62-fc33-4384-9570-4c8d7d2778b9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637de26f46ed2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19873"
    uid: c90cd5d8-8595-464e-b64f-7e44e9a97d7d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18670"
    uid: 1b9a0f56-1418-4247-97b6-fe09195f157f
  kind: Event
  lastTimestamp: "2026-09-17T21:08:49Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:49Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd.18d637de26f4fa34
    namespace: kserve-ci-e2e-test
    resourceVersion: "19875"
    uid: 11c47cd2-ce09-46a0-a104-5ceab308ef85
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74
    namespace: kserve-ci-e2e-test
    resourceVersion: "18659"
    uid: 69254cd1-60bf-4418-bef6-721d067ec819
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: 'Created pod: message-dumper-raw-8d8d6-predictor-7f84fb8b74-tt7nd'
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: message-dumper-raw-8d8d6-predictor-7f84fb8b74.18d637c58fe60676
    namespace: kserve-ci-e2e-test
    resourceVersion: "18671"
    uid: 8e5eb670-ace6-43ff-925a-f1b2efc357f2
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18657"
    uid: b84c6aa2-e31f-4006-80b6-9b220b94f2c7
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: Scaled up replica set message-dumper-raw-8d8d6-predictor-7f84fb8b74 from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: message-dumper-raw-8d8d6-predictor.18d637c58d9fcbf7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18662"
    uid: 078dfd11-b7a2-47fb-b3a4-1df9627160ee
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18658"
    uid: dfeb27c5-a920-4dae-bc4b-471b814067e5
  kind: Event
  lastTimestamp: "2026-09-17T21:07:48Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:48Z"
    name: message-dumper-raw-8d8d6-predictor.18d637c90c10581e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19140"
    uid: d4daec03-15df-46dd-98d4-16120f39bc77
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18658"
    uid: dfeb27c5-a920-4dae-bc4b-471b814067e5
  kind: Event
  lastTimestamp: "2026-09-17T21:07:48Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:07:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:48Z"
    name: message-dumper-raw-8d8d6-predictor.18d637c90c79a74e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19141"
    uid: a0e4c6ed-13d9-4c33-bdcc-8a8c8c1146da
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:33Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18881"
    uid: dfeb27c5-a920-4dae-bc4b-471b814067e5
  kind: Event
  lastTimestamp: "2026-09-17T21:07:33Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:07:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:33Z"
    name: message-dumper-raw-8d8d6-predictor.18d637cc8b45032a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19012"
    uid: d3a24626-799d-4c4d-84a5-1a06094983a2
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:33Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-8d8d6-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18881"
    uid: dfeb27c5-a920-4dae-bc4b-471b814067e5
  kind: Event
  lastTimestamp: "2026-09-17T21:07:33Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:07:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:07:33Z"
    name: message-dumper-raw-8d8d6-predictor.18d637cc8c0ee4f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "19015"
    uid: a6795ab1-319a-4d8e-b77f-01f03d688d16
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18646"
    uid: 609c9d57-ce7f-425e-8446-4e3aba843ab9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:12Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:12Z"
    name: message-dumper-raw-8d8d6.18d637c58da35089
    namespace: kserve-ci-e2e-test
    resourceVersion: "18784"
    uid: 737f483a-f781-4838-9ce7-adc490e238ef
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18646"
    uid: 609c9d57-ce7f-425e-8446-4e3aba843ab9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: 'Failed to update status for InferenceService "message-dumper-raw-8d8d6":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-8d8d6":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: message-dumper-raw-8d8d6.18d637c592fe2f4e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18679"
    uid: 0f81e25c-0109-4b53-8fd1-d700fe038d76
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18646"
    uid: 609c9d57-ce7f-425e-8446-4e3aba843ab9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "message-dumper-raw-8d8d6": the object
    has been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: message-dumper-raw-8d8d6.18d637c592fe83ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "18681"
    uid: 39d59e38-fb37-4514-be77-3de213316d4c
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-8d8d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18782"
    uid: 609c9d57-ce7f-425e-8446-4e3aba843ab9
  kind: Event
  lastTimestamp: "2026-09-17T21:07:12Z"
  message: InferenceService [message-dumper-raw-8d8d6] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:07:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:07:12Z"
    name: message-dumper-raw-8d8d6.18d637c7aa859190
    namespace: kserve-ci-e2e-test
    resourceVersion: "18783"
    uid: 7d72ad72-5500-4546-80e4-cc90c4d07f2e
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:16:51.103042Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24012"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    to ip-10-0-132-201.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:16:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:16:51Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384e5a7c55aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "24016"
    uid: fc9f0bc5-79d3-4163-a169-8119749ebecb
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:51Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:51Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-cipn-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:16:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:51Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384e62e55da5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24025"
    uid: e1ef7f7a-0b1e-40ff-ac61-b04140fd7624
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24019"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: Add eth0 [10.132.0.36/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea04d175b
    namespace: kserve-ci-e2e-test
    resourceVersion: "24037"
    uid: a8e450f5-94d7-4915-871c-380a215336d1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: Container image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea1cd0c6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24040"
    uid: 70ca7c99-bb2f-44fe-b073-4b4b34801ab9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea50e93d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "24041"
    uid: b1ad9c4a-f337-4bd7-85f5-e74083474a37
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea58352a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "24042"
    uid: 4765b0f5-0757-4c82-8af0-99b7c15f79e7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea58d6e89
    namespace: kserve-ci-e2e-test
    resourceVersion: "24043"
    uid: ca4b41b7-9650-445a-82a9-23492403733b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea93bb084
    namespace: kserve-ci-e2e-test
    resourceVersion: "24045"
    uid: 2ecb7d84-13ff-45d6-830c-f009ff46b44b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:16:52Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:16:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:16:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6384ea9ae40ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "24046"
    uid: e302c6fb-92fe-4baf-b84d-e81c47a2c872
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:18:36Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:37Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d6386702586b43
    namespace: kserve-ci-e2e-test
    resourceVersion: "24916"
    uid: b35e518d-dc92-44ca-80e1-8bca2de6bcdf
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:18:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n
    namespace: kserve-ci-e2e-test
    resourceVersion: "24014"
    uid: dd99d6bd-8a68-4afb-8254-bf595ae781a3
  kind: Event
  lastTimestamp: "2026-09-17T21:18:36Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:18:37Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n.18d638670258ed08
    namespace: kserve-ci-e2e-test
    resourceVersion: "24917"
    uid: 4d4e17c3-4e1e-4fad-9ef9-4fc017e7868e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-201.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-201.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-cipn-predictor-9774bf55d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24001"
    uid: e58cb59a-5008-4c44-9f4f-5768946dac8e
  kind: Event
  lastTimestamp: "2026-09-17T21:16:51Z"
  message: 'Created pod: message-dumper-raw-cipn-predictor-9774bf55d-8gg4n'
  metadata:
    creationTimestamp: "2026-09-17T21:16:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:16:51Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d.18d6384e5a08c304
    namespace: kserve-ci-e2e-test
    resourceVersion: "24013"
    uid: bc3d2dbe-529f-48dc-bd8c-99b179173202
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23999"
    uid: a5061698-1e31-4844-ac00-07ed2e8b08cc
  kind: Event
  lastTimestamp: "2026-09-17T21:16:51Z"
  message: Scaled up replica set message-dumper-raw-cipn-predictor-9774bf55d from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:16:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:16:51Z"
    name: message-dumper-raw-cipn-predictor.18d6384e561f7483
    namespace: kserve-ci-e2e-test
    resourceVersion: "24004"
    uid: be8fdbda-e643-4404-8334-101d61639c4e
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24000"
    uid: 0fe1baa9-d3f5-4c2a-a170-77ff73801aba
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: message-dumper-raw-cipn-predictor.18d63851d53295a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "24203"
    uid: 95fa7f4e-163b-4096-be85-8bc35d3898be
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24000"
    uid: 0fe1baa9-d3f5-4c2a-a170-77ff73801aba
  kind: Event
  lastTimestamp: "2026-09-17T21:17:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:17:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:06Z"
    name: message-dumper-raw-cipn-predictor.18d63851d5cbd484
    namespace: kserve-ci-e2e-test
    resourceVersion: "24205"
    uid: aef258a2-5ef0-4d16-9806-3d70e9cc6374
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24204"
    uid: 0fe1baa9-d3f5-4c2a-a170-77ff73801aba
  kind: Event
  lastTimestamp: "2026-09-17T21:17:21Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:17:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:21Z"
    name: message-dumper-raw-cipn-predictor.18d6385554919699
    namespace: kserve-ci-e2e-test
    resourceVersion: "24333"
    uid: 0af6ea2d-3aa1-46a4-9f55-c18c4a7175ad
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24204"
    uid: 0fe1baa9-d3f5-4c2a-a170-77ff73801aba
  kind: Event
  lastTimestamp: "2026-09-17T21:17:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:17:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:17:21Z"
    name: message-dumper-raw-cipn-predictor.18d638555512f2e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24335"
    uid: 3d1b771f-83af-44cf-a1c3-6f892fb2ce3d
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23993"
    uid: 8e7b158b-2150-4be4-aff3-570dd2fd4381
  kind: Event
  lastTimestamp: "2026-09-17T21:17:00Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:16:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:17:00Z"
    name: message-dumper-raw-cipn.18d6384e55d441cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24106"
    uid: 0bccff34-ec5d-4bab-ab1d-b52b2640362d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:16:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23993"
    uid: 8e7b158b-2150-4be4-aff3-570dd2fd4381
  kind: Event
  lastTimestamp: "2026-09-17T21:16:51Z"
  message: 'Failed to update status for InferenceService "message-dumper-raw-cipn":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cipn":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:16:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:16:51Z"
    name: message-dumper-raw-cipn.18d6384e5636e7d8
    namespace: kserve-ci-e2e-test
    resourceVersion: "24005"
    uid: cd424910-e0d3-4016-87ab-c597d5c4e930
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:17:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24104"
    uid: 8e7b158b-2150-4be4-aff3-570dd2fd4381
  kind: Event
  lastTimestamp: "2026-09-17T21:17:00Z"
  message: InferenceService [message-dumper-raw-cipn] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:17:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:17:00Z"
    name: message-dumper-raw-cipn.18d6385071f59aa4
    namespace: kserve-ci-e2e-test
    resourceVersion: "24105"
    uid: 9a330a67-7162-464f-b0f6-524c667a6163
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:04:52.899218Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17195"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-669c3-797fc96956-ftdzs
    to ip-10-0-131-164.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:04:52Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:04:52Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a722339adf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17200"
    uid: 139e0415-c0bc-451b-93c4-215abfa40ca4
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:53Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17204"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:53Z"
  message: Add eth0 [10.134.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:04:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:04:53Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a74434188a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17217"
    uid: 03c0593a-9570-4ba1-b1b3-39bb5a5ddabe
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-669c3}
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17197"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:53Z"
  message: Pulling image "quay.io/opendatahub/kserve-router@sha256:2c5833579293a5043b5db03d2dc3bee6dec8d8c7a284a9a790353f02c7a72dd3"
  metadata:
    creationTimestamp: "2026-09-17T21:04:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:53Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a7456e8739
    namespace: kserve-ci-e2e-test
    resourceVersion: "17219"
    uid: c070e2a6-6c7c-4fab-8d9d-27e9f6b7d8b6
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-669c3}
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17197"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:55Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:2c5833579293a5043b5db03d2dc3bee6dec8d8c7a284a9a790353f02c7a72dd3"
    in 2.029s (2.029s including waiting). Image size: 211751499 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T21:04:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:55Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a7be5fc761
    namespace: kserve-ci-e2e-test
    resourceVersion: "17231"
    uid: 37a6d334-4b44-4576-88ab-e491a3d89bd2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-669c3}
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17197"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:55Z"
  message: 'Created container: model-chainer-raw-669c3'
  metadata:
    creationTimestamp: "2026-09-17T21:04:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:55Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a7c140c30d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17232"
    uid: 6792c03f-0904-4b36-92ba-a263bb942f53
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-669c3}
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17197"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:55Z"
  message: Started container model-chainer-raw-669c3
  metadata:
    creationTimestamp: "2026-09-17T21:04:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:04:55Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a7c1b43669
    namespace: kserve-ci-e2e-test
    resourceVersion: "17233"
    uid: c020ea00-e8f0-442a-bc86-155e84d3df3c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-669c3}
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17197"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:05:02Z"
  message: Stopping container model-chainer-raw-669c3
  metadata:
    creationTimestamp: "2026-09-17T21:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:02Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637a97a9746dd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17318"
    uid: cbe69e22-7a89-4380-9a60-0424f398c448
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-17T21:05:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-669c3}
    kind: Pod
    name: model-chainer-raw-669c3-797fc96956-ftdzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "17197"
    uid: d6438e48-a4ca-406e-ba79-fe38b72c480b
  kind: Event
  lastTimestamp: "2026-09-17T21:05:31Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T21:05:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:05:31Z"
    name: model-chainer-raw-669c3-797fc96956-ftdzs.18d637aa64ac760e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17702"
    uid: 413a149c-6c64-45f7-b379-7060bf4dbc25
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:52Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-669c3-797fc96956
    namespace: kserve-ci-e2e-test
    resourceVersion: "17192"
    uid: 2b99c309-6ce1-476b-b87d-1171ef23786b
  kind: Event
  lastTimestamp: "2026-09-17T21:04:52Z"
  message: 'Created pod: model-chainer-raw-669c3-797fc96956-ftdzs'
  metadata:
    creationTimestamp: "2026-09-17T21:04:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:52Z"
    name: model-chainer-raw-669c3-797fc96956.18d637a721a861b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17196"
    uid: 58dd87ed-873c-44f1-89ee-5a01557eb7d0
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:52Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17190"
    uid: 0f1c51d5-c5f7-4148-a85b-28a8c9ed37a5
  kind: Event
  lastTimestamp: "2026-09-17T21:04:52Z"
  message: Scaled up replica set model-chainer-raw-669c3-797fc96956 from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:04:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:04:52Z"
    name: model-chainer-raw-669c3.18d637a72102abac
    namespace: kserve-ci-e2e-test
    resourceVersion: "17193"
    uid: 8b272b0f-9b9c-4fd2-a9a2-cda5896a329a
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:04:52Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-669c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17206"
    uid: 6ea45eb7-c9ab-461a-9408-60ab3dacfc5f
  kind: Event
  lastTimestamp: "2026-09-17T21:04:52Z"
  message: InferenceGraph [model-chainer-raw-669c3] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:04:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:04:52Z"
    name: model-chainer-raw-669c3.18d637a7263204a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17208"
    uid: 164d321b-61b7-4af0-8cc2-002935db3518
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:06:53.228739Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18484"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    to ip-10-0-131-164.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c326666bb2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18490"
    uid: ad0adbc4-b88a-42be-9fa1-3e9756ff05b1
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18493"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: Add eth0 [10.134.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c3486f4c46
    namespace: kserve-ci-e2e-test
    resourceVersion: "18508"
    uid: 97723239-5df0-4ccc-8735-d897346e093f
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e3e0d}
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18488"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: Container image "quay.io/opendatahub/kserve-router@sha256:2c5833579293a5043b5db03d2dc3bee6dec8d8c7a284a9a790353f02c7a72dd3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c349c6674a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18510"
    uid: b6b914cf-eefa-48c9-8f0f-7bdbc7da23ac
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e3e0d}
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18488"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: 'Created container: model-chainer-raw-hpa-e3e0d'
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c34c6141ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "18511"
    uid: 2912a5a1-a348-4b37-a2d9-f2f02e5d5871
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e3e0d}
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18488"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: Started container model-chainer-raw-hpa-e3e0d
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c34ccdc128
    namespace: kserve-ci-e2e-test
    resourceVersion: "18512"
    uid: d047d4c2-d68e-4d93-abf8-d340f4d9bdbc
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e3e0d}
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18488"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: "2026-09-17T21:07:03Z"
  message: Stopping container model-chainer-raw-hpa-e3e0d
  metadata:
    creationTimestamp: "2026-09-17T21:07:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:03Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c57f74af55
    namespace: kserve-ci-e2e-test
    resourceVersion: "18608"
    uid: 5a86aefe-59fa-4381-8959-cab8200afe9f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-17T21:07:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e3e0d}
    kind: Pod
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18488"
    uid: 39d03ce4-9be9-4161-b2e4-c15c8fb9a817
  kind: Event
  lastTimestamp: "2026-09-17T21:07:30Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T21:07:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:07:30Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5.18d637c5fb0f99bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "18990"
    uid: fa4195fe-9bf2-4bd9-89c0-a097ae0b8b12
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-131-164.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-131-164.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18481"
    uid: 19944238-fcf9-4514-a3a0-5ba5b37cad48
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: 'Created pod: model-chainer-raw-hpa-e3e0d-64dbcbcbc6-z9hr5'
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d-64dbcbcbc6.18d637c325a79e60
    namespace: kserve-ci-e2e-test
    resourceVersion: "18486"
    uid: ef7d0972-7bbc-4a1e-8da3-9c30c4bba23c
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18479"
    uid: cd289cb7-02ed-4e08-8e6c-3d217ca2d355
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: Scaled up replica set model-chainer-raw-hpa-e3e0d-64dbcbcbc6 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d.18d637c324a40dbf
    namespace: kserve-ci-e2e-test
    resourceVersion: "18482"
    uid: 1180af5a-85cb-4ce6-a51c-d67c2bce7beb
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18495"
    uid: 023c0cc8-bb44-4976-a645-ea41cbaf3a7c
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: InferenceGraph [model-chainer-raw-hpa-e3e0d] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d.18d637c32a167ca2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18497"
    uid: 71445f36-1feb-4bdc-b209-a9b6597af435
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18473"
    uid: 023c0cc8-bb44-4976-a645-ea41cbaf3a7c
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: 'Failed to update status for InferenceGraph "model-chainer-raw-hpa-e3e0d":
    Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-e3e0d":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d.18d637c32c386117
    namespace: kserve-ci-e2e-test
    resourceVersion: "18500"
    uid: 4a6e8aa6-c664-49d2-a447-4f20fbc26b0a
  reason: UpdateFailed
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:06:53Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-e3e0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18473"
    uid: 023c0cc8-bb44-4976-a645-ea41cbaf3a7c
  kind: Event
  lastTimestamp: "2026-09-17T21:06:53Z"
  message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on
    inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-e3e0d": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:06:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:06:53Z"
    name: model-chainer-raw-hpa-e3e0d.18d637c32c38c417
    namespace: kserve-ci-e2e-test
    resourceVersion: "18501"
    uid: c9d0dd8b-fa2c-4721-94fb-c2d720cbad5b
  reason: InternalError
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:08:39.567660Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19727"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:08:39Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:08:39Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dbe8b18d20
    namespace: kserve-ci-e2e-test
    resourceVersion: "19734"
    uid: 134a862b-c11e-44a2-83fa-f22b447a59be
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:40Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19740"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:40Z"
  message: Add eth0 [10.133.0.36/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:08:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:08:40Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dc0a8ae848
    namespace: kserve-ci-e2e-test
    resourceVersion: "19747"
    uid: 8203f419-88fc-46f6-ad7c-fc7ee8487f34
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:40Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:40Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dc0bd75c55
    namespace: kserve-ci-e2e-test
    resourceVersion: "19749"
    uid: 40912e61-8d1f-4ede-8047-8167bb57c3c3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:40Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:08:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:40Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dc0edba8a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "19750"
    uid: 21627394-35ed-45e9-b0a4-4f132416f1d1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:40Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:08:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:40Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dc0f4f8c74
    namespace: kserve-ci-e2e-test
    resourceVersion: "19751"
    uid: 5003f079-ab88-42c6-a10b-493540603267
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:44Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:44Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dcfc6da49b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19784"
    uid: aeb6a37c-96c6-4d07-b89a-af39abcbe40c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:44Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:08:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:44Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dcff3a6b86
    namespace: kserve-ci-e2e-test
    resourceVersion: "19786"
    uid: 2e82a085-b955-4e72-a9b5-443619dab853
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:44Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:08:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:44Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dcffa90813
    namespace: kserve-ci-e2e-test
    resourceVersion: "19788"
    uid: e90b61c7-e185-44ec-ab65-900e141ff691
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:44Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:08:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:44Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dcffb26f06
    namespace: kserve-ci-e2e-test
    resourceVersion: "19790"
    uid: 9c92556d-0280-4ad7-bcaa-f719b245404c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:44Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:08:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:44Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dd02c14a7b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19792"
    uid: 2bb0051c-d543-49d2-87da-094636fc84fd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:08:44Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:08:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:08:44Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dd03324987
    namespace: kserve-ci-e2e-test
    resourceVersion: "19793"
    uid: d1669331-7654-44a7-8cbe-2fa76e078662
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:10:11Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:08:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:11Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637dd386d324f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20582"
    uid: 41b69277-1259-4393-9043-9efc549dbfb3
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:10:10Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:10Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637f0fc7231d8
    namespace: kserve-ci-e2e-test
    resourceVersion: "20568"
    uid: 411dc91d-8556-4c8f-838e-609db8db73c1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:10:10Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:10Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637f0fc72af47
    namespace: kserve-ci-e2e-test
    resourceVersion: "20569"
    uid: df7c2d5f-a43e-44c9-846b-0987fdcbeb57
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z
    namespace: kserve-ci-e2e-test
    resourceVersion: "19731"
    uid: b674185e-798a-46b1-ae2b-4d7a0c9e9c82
  kind: Event
  lastTimestamp: "2026-09-17T21:10:11Z"
  message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp
    10.133.0.36:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:10:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:11Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z.18d637f13e960c8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20581"
    uid: b4c70de3-e199-4146-9739-571d061d914c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:39Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-a9ce0-predictor-948cdbc56
    namespace: kserve-ci-e2e-test
    resourceVersion: "19718"
    uid: cef983ea-080a-4393-94de-784005368c93
  kind: Event
  lastTimestamp: "2026-09-17T21:08:39Z"
  message: 'Created pod: raw-sklearn-a9ce0-predictor-948cdbc56-d5d8z'
  metadata:
    creationTimestamp: "2026-09-17T21:08:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:39Z"
    name: raw-sklearn-a9ce0-predictor-948cdbc56.18d637dbe8307361
    namespace: kserve-ci-e2e-test
    resourceVersion: "19730"
    uid: 02edfbfc-5791-4bdd-836f-c3c0fb4bc0ac
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:39Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-a9ce0-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19716"
    uid: 639b6111-4f44-46e5-88ad-caec41f6ee75
  kind: Event
  lastTimestamp: "2026-09-17T21:08:39Z"
  message: Scaled up replica set raw-sklearn-a9ce0-predictor-948cdbc56 from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:08:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:08:39Z"
    name: raw-sklearn-a9ce0-predictor.18d637dbe555c3cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19720"
    uid: fcdfc473-c0d6-4069-870c-e53546a51bed
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:54Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-a9ce0-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19717"
    uid: d54c7e65-4938-45be-9064-15507843b143
  kind: Event
  lastTimestamp: "2026-09-17T21:09:24Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:08:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:09:24Z"
    name: raw-sklearn-a9ce0-predictor.18d637df63d812e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20215"
    uid: 63feee80-0a05-426f-aed3-d5cc575ded34
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:54Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-a9ce0-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19717"
    uid: d54c7e65-4938-45be-9064-15507843b143
  kind: Event
  lastTimestamp: "2026-09-17T21:09:24Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:08:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:09:24Z"
    name: raw-sklearn-a9ce0-predictor.18d637df644b925c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20216"
    uid: 61411549-26fc-4120-950a-24426502c0db
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:09:39Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-a9ce0-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19966"
    uid: d54c7e65-4938-45be-9064-15507843b143
  kind: Event
  lastTimestamp: "2026-09-17T21:10:09Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:09:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:10:09Z"
    name: raw-sklearn-a9ce0-predictor.18d637e9e0be5391
    namespace: kserve-ci-e2e-test
    resourceVersion: "20547"
    uid: 9e3ffd6c-f532-4085-8168-b6675c09274a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T21:09:39Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-a9ce0-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19966"
    uid: d54c7e65-4938-45be-9064-15507843b143
  kind: Event
  lastTimestamp: "2026-09-17T21:10:09Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:09:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:10:09Z"
    name: raw-sklearn-a9ce0-predictor.18d637e9e129d332
    namespace: kserve-ci-e2e-test
    resourceVersion: "20548"
    uid: 62c475f1-626f-421d-9876-4ec7c984ecf5
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:39Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-a9ce0
    namespace: kserve-ci-e2e-test
    resourceVersion: "19709"
    uid: 5b6d3329-72b8-4a8b-91b7-5841cf17c8bb
  kind: Event
  lastTimestamp: "2026-09-17T21:10:01Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:08:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:10:01Z"
    name: raw-sklearn-a9ce0.18d637dbe553e13d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20490"
    uid: 9c84c1cb-9a5f-4e3a-8cb3-aa871c095698
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:08:39Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-a9ce0
    namespace: kserve-ci-e2e-test
    resourceVersion: "19709"
    uid: 5b6d3329-72b8-4a8b-91b7-5841cf17c8bb
  kind: Event
  lastTimestamp: "2026-09-17T21:08:39Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-a9ce0": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-a9ce0":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:08:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:08:39Z"
    name: raw-sklearn-a9ce0.18d637dbe5e8508e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19724"
    uid: 6bc2fae0-3b9d-49aa-bc83-cee85a5fc319
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-a9ce0
    namespace: kserve-ci-e2e-test
    resourceVersion: "20488"
    uid: 5b6d3329-72b8-4a8b-91b7-5841cf17c8bb
  kind: Event
  lastTimestamp: "2026-09-17T21:10:01Z"
  message: InferenceService [raw-sklearn-a9ce0] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:10:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:10:01Z"
    name: raw-sklearn-a9ce0.18d637eeefead3ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "20489"
    uid: e0382ce5-56ba-48cc-9fbf-273fbd444d4e
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T21:10:15.291531Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20638"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    to ip-10-0-138-140.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T21:10:15Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T21:10:15Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f23247f717
    namespace: kserve-ci-e2e-test
    resourceVersion: "20643"
    uid: 0243105f-f773-40c2-b49a-db2a77609488
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-b97d769c4-wmktk
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:15Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:15Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-5d5fc-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T21:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:15Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f23f0c47b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20651"
    uid: d6ee7904-e9cd-4d2f-81ce-6ad1b72872dc
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:16Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20647"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:16Z"
  message: Add eth0 [10.133.0.37/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T21:10:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T21:10:16Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f27848e8d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "20664"
    uid: 6f4d5758-1791-4f84-9a51-e9a8a17a61fd
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:16Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:088810e21187103d825698143fc93d3ba94ca1f9b7facd7cf4b1d40af2e9f6fb"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:10:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:16Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f27a141cdc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20666"
    uid: c09e1f6e-f2a5-4e75-9858-915b9c65a300
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:16Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T21:10:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:16Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f27d73dba6
    namespace: kserve-ci-e2e-test
    resourceVersion: "20667"
    uid: 6c688841-cf5f-43b3-802b-b0e9c548bcb8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:16Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T21:10:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:16Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f27ddff4a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "20668"
    uid: d55951ca-1957-49a8-ba6c-96ec14873906
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:20Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1999"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:10:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:20Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f36e11ae8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20703"
    uid: 20bb0ac1-0305-4ae0-ab4c-efea1b7e4001
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:20Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T21:10:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:20Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f3714cabbb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20705"
    uid: d48b1670-15d7-4b3e-b3c2-76e2d622b0a5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:20Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:10:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:20Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f371bb50ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "20706"
    uid: dc98891f-75ef-446d-957c-d89d915da92b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:20Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T21:10:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:20Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f371c5e093
    namespace: kserve-ci-e2e-test
    resourceVersion: "20707"
    uid: e9ab1807-6643-42f3-a0e5-a7962b40267e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:20Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T21:10:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:20Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f3748709f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "20710"
    uid: cb9d8629-46eb-427a-b8f1-051a157a5683
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:10:20Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:10:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:10:20Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f374f8e280
    namespace: kserve-ci-e2e-test
    resourceVersion: "20712"
    uid: 76e77127-51f4-4d13-a48a-a1ae7a495bff
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:11:48Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T21:10:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:48Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d637f3e63ad8cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21498"
    uid: 4ac9b069-0628-46c0-ba74-d8be27cefb69
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d63807578b0cf7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21444"
    uid: 4c49c383-37d8-4eb0-8533-eddc071e2750
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:11:46Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T21:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:46Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d63807578bb72b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21446"
    uid: 70c0caae-9680-48f3-a98e-77834ee23084
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x
    namespace: kserve-ci-e2e-test
    resourceVersion: "20641"
    uid: 71503f3a-f40a-4276-aac9-d08b2d976d4b
  kind: Event
  lastTimestamp: "2026-09-17T21:11:48Z"
  message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp
    10.133.0.37:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T21:11:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T21:11:48Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x.18d63807ec6c2bf0
    namespace: kserve-ci-e2e-test
    resourceVersion: "21497"
    uid: e1bc2e56-5dd6-4b1b-9b18-3459c6eb39e7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-138-140.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-138-140.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:15Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20626"
    uid: d1113e7d-1f14-4db3-8d89-a55875133522
  kind: Event
  lastTimestamp: "2026-09-17T21:10:15Z"
  message: 'Created pod: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b-bfq9x'
  metadata:
    creationTimestamp: "2026-09-17T21:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:10:15Z"
    name: raw-sklearn-runtime-5d5fc-predictor-5b8446f67b.18d637f231d2c6f2
    namespace: kserve-ci-e2e-test
    resourceVersion: "20640"
    uid: 3e3ddfd5-d917-4fd6-afc2-d9265659ab97
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:15Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-runtime-5d5fc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20624"
    uid: db50906c-82bd-4572-b90f-ed48abc4ab62
  kind: Event
  lastTimestamp: "2026-09-17T21:10:15Z"
  message: Scaled up replica set raw-sklearn-runtime-5d5fc-predictor-5b8446f67b from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T21:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:10:15Z"
    name: raw-sklearn-runtime-5d5fc-predictor.18d637f22e8e2d48
    namespace: kserve-ci-e2e-test
    resourceVersion: "20627"
    uid: 7e55cc85-ba03-4717-8ef6-b762911a2f35
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:30Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5d5fc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20625"
    uid: 2552e1cc-d8be-41e0-9f2b-e51fb04f5f99
  kind: Event
  lastTimestamp: "2026-09-17T21:10:45Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:10:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:10:45Z"
    name: raw-sklearn-runtime-5d5fc-predictor.18d637f5ad105672
    namespace: kserve-ci-e2e-test
    resourceVersion: "20892"
    uid: 8fae3415-3ccb-42f1-b6bf-b86bd986c1bf
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:30Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5d5fc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20625"
    uid: 2552e1cc-d8be-41e0-9f2b-e51fb04f5f99
  kind: Event
  lastTimestamp: "2026-09-17T21:10:45Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T21:10:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:10:45Z"
    name: raw-sklearn-runtime-5d5fc-predictor.18d637f5ad786c89
    namespace: kserve-ci-e2e-test
    resourceVersion: "20894"
    uid: 71ce2732-9d45-4706-9559-f717586d005d
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:00Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5d5fc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20783"
    uid: 2552e1cc-d8be-41e0-9f2b-e51fb04f5f99
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:11:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: raw-sklearn-runtime-5d5fc-predictor.18d637fcab10c412
    namespace: kserve-ci-e2e-test
    resourceVersion: "21358"
    uid: a9feaef3-4528-4ed0-8ea6-83f233793f51
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:00Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5d5fc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20783"
    uid: 2552e1cc-d8be-41e0-9f2b-e51fb04f5f99
  kind: Event
  lastTimestamp: "2026-09-17T21:11:45Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T21:11:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T21:11:45Z"
    name: raw-sklearn-runtime-5d5fc-predictor.18d637fcab82b758
    namespace: kserve-ci-e2e-test
    resourceVersion: "21359"
    uid: cde4371b-ba93-4581-a9c6-9d5e785e227d
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:15Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5d5fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20615"
    uid: 6135a71e-a64e-4109-9552-2f5b6a004e42
  kind: Event
  lastTimestamp: "2026-09-17T21:11:38Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T21:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:11:38Z"
    name: raw-sklearn-runtime-5d5fc.18d637f22e8f22e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "21295"
    uid: fd96e15e-9aec-43bc-955d-47acbbb7dffe
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:10:15Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5d5fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20615"
    uid: 6135a71e-a64e-4109-9552-2f5b6a004e42
  kind: Event
  lastTimestamp: "2026-09-17T21:10:15Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-runtime-5d5fc":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-5d5fc":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T21:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:10:15Z"
    name: raw-sklearn-runtime-5d5fc.18d637f22f1ae781
    namespace: kserve-ci-e2e-test
    resourceVersion: "20632"
    uid: 0ce018c6-6054-461d-8a65-93a7f0299179
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T21:11:38Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5d5fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "21293"
    uid: 6135a71e-a64e-4109-9552-2f5b6a004e42
  kind: Event
  lastTimestamp: "2026-09-17T21:11:38Z"
  message: InferenceService [raw-sklearn-runtime-5d5fc] is Ready
  metadata:
    creationTimestamp: "2026-09-17T21:11:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T21:11:38Z"
    name: raw-sklearn-runtime-5d5fc.18d638059d50ca5a
    namespace: kserve-ci-e2e-test
    resourceVersion: "21294"
    uid: 49eac2c5-488b-4b98-8dd3-fb27d86cf02a
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
kind: EventList
metadata:
  resourceVersion: "25168"