---
apiVersion: v1
items:
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:38:02.738827Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19058"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-cb85cf55c-sphmz
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817e878a64a8
    namespace: kserve-ci-e2e-test
    resourceVersion: "19064"
    uid: a68931ad-a0aa-437b-acfb-c731fa170aba
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-create-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817e93760c57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19086"
    uid: 4c4ed476-f02d-430c-9645-182e6beaa939
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19074"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: Add eth0 [10.134.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817ec4c9c494
    namespace: kserve-ci-e2e-test
    resourceVersion: "19105"
    uid: 45cc190e-676b-4279-986f-8ad6a99bfccb
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817ec62ad5ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "19107"
    uid: 23a598fd-8d97-4494-b479-4d13351b9bad
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817ec92dae78
    namespace: kserve-ci-e2e-test
    resourceVersion: "19109"
    uid: 6f0782b5-473d-4495-ae8a-1437b245f20e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817ec9a023d8
    namespace: kserve-ci-e2e-test
    resourceVersion: "19110"
    uid: 99d59deb-5a80-45f9-8c1e-f058305f6688
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:08Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:08Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817fd9ba72e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19163"
    uid: 6849daee-5a84-488e-b349-194d101130b8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:08Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:08Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817fdc48a3bc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19167"
    uid: 63e61f04-c81c-47eb-89b9-225e070cc65f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:08Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:08Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817fdcb7ccda
    namespace: kserve-ci-e2e-test
    resourceVersion: "19168"
    uid: eeb912e8-b89f-4a3f-9e11-b6a5beefb8f5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:08Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:08Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817fdcc12122
    namespace: kserve-ci-e2e-test
    resourceVersion: "19169"
    uid: 884334f9-c282-483a-ab2a-e660c81aab56
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:08Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:08Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817fdf83e3e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19172"
    uid: 4ef0a29e-0412-4645-852d-094340cb2daf
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:38:08Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:08Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6817fdff5352b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19173"
    uid: cf9fcd13-0318-4522-8466-91d1d35e4474
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:39:06Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:38:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:06Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c6818051b4ddb3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19589"
    uid: 0ee669e0-bb9c-479a-a499-672e660a085c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:39:27Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:39:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:27Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c681925746b9fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "19768"
    uid: fa6c912f-3080-4d25-8781-7f73fb4847f3
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:39:27Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:39:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:27Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c681925747214b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19769"
    uid: 32158909-494f-4662-b2b1-b79cc99f5898
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-sphmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "19060"
    uid: d9d6bf13-47e1-4d3d-88e0-edfb35aa6675
  kind: Event
  lastTimestamp: "2026-07-28T16:39:31Z"
  message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp
    10.134.0.27:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:39:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:31Z"
    name: isvc-canary-create-predictor-cb85cf55c-sphmz.18c681932dd8f713
    namespace: kserve-ci-e2e-test
    resourceVersion: "19804"
    uid: 7c4e6c03-fba5-4090-aef4-6bc9c87ecba7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-cb85cf55c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19042"
    uid: aea40e04-a04d-4cd9-95f6-fefb2662dcdd
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: 'Created pod: isvc-canary-create-predictor-cb85cf55c-sphmz'
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-predictor-cb85cf55c.18c6817e870f8016
    namespace: kserve-ci-e2e-test
    resourceVersion: "19059"
    uid: ef4e0d35-7508-47aa-8852-48764f9bbbbd
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19041"
    uid: 1e492454-e4fa-47f5-af50-e81cf9fb7a6c
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: Scaled up replica set isvc-canary-create-predictor-cb85cf55c from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-predictor.18c6817e820050e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19043"
    uid: fa9d04ba-9bc1-4c3d-a72b-15a364915970
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:38:02.756291Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19062"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817e8894d82f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19069"
    uid: 0732d0f2-c8ca-4217-ac8e-e3bc07558d8c
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19077"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: Add eth0 [10.134.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817ea1cb4c8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19093"
    uid: 18f6fff8-388b-422d-9d61-204467c673f6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817ea37ae5d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19096"
    uid: 9344b670-0206-459c-bb2c-2a0f0f34848d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817ea5e04373
    namespace: kserve-ci-e2e-test
    resourceVersion: "19097"
    uid: e5872fec-4f7a-44b1-908a-684f9aa88b89
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:03Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:38:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817ea6974398
    namespace: kserve-ci-e2e-test
    resourceVersion: "19098"
    uid: 9ad7122d-6405-4c34-a213-a03455c971c1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:38:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817f9dd785f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19142"
    uid: f8d7be23-8f32-44e1-a73b-e8d9ffe083d0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:38:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817fa1c6ef14
    namespace: kserve-ci-e2e-test
    resourceVersion: "19146"
    uid: e4cb1ab3-6f25-4579-94ba-130af7baa1db
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:38:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817fa238dc07
    namespace: kserve-ci-e2e-test
    resourceVersion: "19147"
    uid: 885a69da-9add-4b81-a014-7323a454ec0a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:38:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817fa2429853
    namespace: kserve-ci-e2e-test
    resourceVersion: "19148"
    uid: cd795bf0-ccff-4abd-aacf-731988f2a4aa
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:38:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817fa5b97bc3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19150"
    uid: 87048651-4723-4bf2-81b1-458f53134b7c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:38:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:38:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817fa6303bfb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19151"
    uid: b9bc40e1-144c-4fa9-995e-f57eaad7a75d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:39:14Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:38:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:14Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6817fd9e67b7a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19641"
    uid: 28d78250-5a7e-4b23-963a-66d681acd495
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:39:27Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:39:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:27Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6819257cd47ab
    namespace: kserve-ci-e2e-test
    resourceVersion: "19770"
    uid: 7ac50399-8415-4121-a634-d43d07ec4ce9
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:39:27Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:39:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:27Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c6819257ce7c57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19771"
    uid: 4ec8d5e2-6a22-4c08-a4dd-3b25139ab624
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 432a333f-0f95-430a-8efa-2b1ce8be5ebc
  kind: Event
  lastTimestamp: "2026-07-28T16:39:29Z"
  message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp
    10.134.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:39:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:29Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9.18c68192b6374575
    namespace: kserve-ci-e2e-test
    resourceVersion: "19786"
    uid: 02613550-a22a-4ce4-bf6a-2a1d98200e47
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-v2-predictor-6d8d4768d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19051"
    uid: 2ed1aa30-bdbb-4d39-8385-7994c0be61a3
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: 'Created pod: isvc-canary-create-v2-predictor-6d8d4768d4-xh5s9'
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4.18c6817e87df76ab
    namespace: kserve-ci-e2e-test
    resourceVersion: "19065"
    uid: 4ebd9ec3-14bf-4a71-945d-e5f3a67019bd
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19050"
    uid: ed08be78-17b1-4143-897d-ada2dd67bb61
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: Scaled up replica set isvc-canary-create-v2-predictor-6d8d4768d4 from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create-v2-predictor.18c6817e842528f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "19053"
    uid: 2e0b176a-57fe-4a25-a0e1-7c928ce5b1a7
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 17
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "19033"
    uid: 66f96a84-8ac3-4d74-af02-da7d8d3f502d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:24Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:39:24Z"
    name: isvc-canary-create.18c6817e83d0b8f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19722"
    uid: ffe256c4-616a-4474-8c0b-51e13d78f7ae
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "19033"
    uid: 66f96a84-8ac3-4d74-af02-da7d8d3f502d
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: 'Failed to update status for InferenceService "isvc-canary-create": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create.18c6817e844bab0e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19080"
    uid: c53afd4e-ad22-4234-8ddc-c59c18f60d09
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:38:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "19033"
    uid: 66f96a84-8ac3-4d74-af02-da7d8d3f502d
  kind: Event
  lastTimestamp: "2026-07-28T16:38:02Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:38:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:38:02Z"
    name: isvc-canary-create.18c6817e844be8f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19081"
    uid: 00d3bf0f-b1bc-45f9-82cd-585de82c28c4
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:16Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "19660"
    uid: 66f96a84-8ac3-4d74-af02-da7d8d3f502d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:16Z"
  message: InferenceService [isvc-canary-create] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:39:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:39:16Z"
    name: isvc-canary-create.18c6818fb842ec0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19661"
    uid: 047fa8bc-ea76-41ae-afac-7d7391ba899c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:39:33.027242Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19850"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-d9759ffd5-ppfrd
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c681938d263f01
    namespace: kserve-ci-e2e-test
    resourceVersion: "19859"
    uid: c7f5a7a3-6aa5-405c-b15f-d10a7d1e279d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19869"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: Add eth0 [10.134.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68193a68df136
    namespace: kserve-ci-e2e-test
    resourceVersion: "19884"
    uid: e8841f74-7f02-457a-ba56-b9741872ce75
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68193a7f42251
    namespace: kserve-ci-e2e-test
    resourceVersion: "19887"
    uid: e95b3b9c-9be2-4b3e-b4e5-a1c75d6544d5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68193ab57d7e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "19888"
    uid: 223b1d59-12e7-4ea3-8f1f-76fa7765b8d1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68193abc7ffab
    namespace: kserve-ci-e2e-test
    resourceVersion: "19889"
    uid: 19528cc4-1f2c-4207-b51a-22141cd6470b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:37Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:39:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:37Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194a89d8591
    namespace: kserve-ci-e2e-test
    resourceVersion: "19932"
    uid: d65894d9-7012-4b31-93ec-71a732d553c6
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:37Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:39:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:37Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194af2d0367
    namespace: kserve-ci-e2e-test
    resourceVersion: "19936"
    uid: a2957af0-283d-413f-b313-0464d60a1d19
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:37Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:39:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:37Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194afa59e60
    namespace: kserve-ci-e2e-test
    resourceVersion: "19937"
    uid: 055b68cf-17a6-4faa-809f-2605d00bfccd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:37Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:39:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:37Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194afb16834
    namespace: kserve-ci-e2e-test
    resourceVersion: "19938"
    uid: ef9e124f-da93-437f-8590-4bf762a810fd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:37Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:39:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:37Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194b295f7f1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19940"
    uid: 93707e0c-ab71-4d66-87cc-d52d03c64bbe
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:39:37Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:39:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:37Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194b3098694
    namespace: kserve-ci-e2e-test
    resourceVersion: "19941"
    uid: e41db58e-ae73-48cd-bbee-a9a037c87597
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:41:04Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:04Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c68194e4a1d772
    namespace: kserve-ci-e2e-test
    resourceVersion: "20611"
    uid: 1e34d8e9-6e5b-4070-a69c-9ffca756db4e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c681a8878c2152
    namespace: kserve-ci-e2e-test
    resourceVersion: "20584"
    uid: 3f4312ba-2c78-4b4a-990a-ebcf45703862
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c681a8878cb7c7
    namespace: kserve-ci-e2e-test
    resourceVersion: "20585"
    uid: c568e319-b56d-4595-8e27-697345271754
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: c29968a6-cf97-4425-bb03-79940e2af98d
  kind: Event
  lastTimestamp: "2026-07-28T16:41:04Z"
  message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp
    10.134.0.29:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:41:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:04Z"
    name: isvc-canary-promote-predictor-d9759ffd5-ppfrd.18c681a8eb14ad01
    namespace: kserve-ci-e2e-test
    resourceVersion: "20617"
    uid: ce1701e1-3ff2-40e9-958e-4226eaf4af9b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-d9759ffd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "19833"
    uid: 94a6bd21-c7e8-40b7-bf45-0a4c25b58b00
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: 'Created pod: isvc-canary-promote-predictor-d9759ffd5-ppfrd'
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-predictor-d9759ffd5.18c681938c7ade3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19852"
    uid: 4a608c66-d1f6-4683-8155-01b3b9220d18
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:32Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19832"
    uid: c94e03da-bb22-4442-8d20-87b20ce91eb9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:32Z"
  message: Scaled up replica set isvc-canary-promote-predictor-d9759ffd5 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T16:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:39:32Z"
    name: isvc-canary-promote-predictor.18c6819386bd9d55
    namespace: kserve-ci-e2e-test
    resourceVersion: "19834"
    uid: 5d84668a-fed3-4861-a069-6f8ea0d8cc84
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:41:03.201750Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20589"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-86766c4958-g972h
    to ip-10-0-132-37.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681a88bf81850
    namespace: kserve-ci-e2e-test
    resourceVersion: "20595"
    uid: 6fe84cc2-1cc5-479a-afa8-a0bb4ef93394
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20601"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: Add eth0 [10.133.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681a8a5151d23
    namespace: kserve-ci-e2e-test
    resourceVersion: "20605"
    uid: 8b31ee17-db09-48f6-8e44-74b0db655ba2
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681a8a70965cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20607"
    uid: 39450960-00f9-42d8-9609-b9856a62affd
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:07Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    in 3.702s (3.702s including waiting). Image size: 300915674 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:41:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:07Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681a983b470e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20638"
    uid: 638959e3-70d7-4d26-80ae-3c818493574a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:07Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:41:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:07Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681a986cb6f2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "20639"
    uid: 22d767a6-df74-4f17-8c98-b9f62885276c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:07Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:41:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:07Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681a98725811e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20640"
    uid: 8334f9bf-5eed-4feb-ae07-01d193dfb1ed
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:11Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
  metadata:
    creationTimestamp: "2026-07-28T16:41:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:11Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681aa6b360df5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20673"
    uid: 979db539-3259-40ea-9a2f-642b3dbfccc4
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:23Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    in 12.712s (12.712s including waiting). Image size: 1555831221 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:41:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:23Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681ad60ed2816
    namespace: kserve-ci-e2e-test
    resourceVersion: "20764"
    uid: 71f80a7a-b2fb-4bec-8998-5d2e7ef77ba1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:24Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:41:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:24Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681ad63dae518
    namespace: kserve-ci-e2e-test
    resourceVersion: "20766"
    uid: d511e1a3-e16c-42f4-88f7-f9a0817d9ccd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:24Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:41:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:24Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681ad64397be0
    namespace: kserve-ci-e2e-test
    resourceVersion: "20767"
    uid: dd49cde1-1c16-4a65-b64c-9a8201479c64
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:24Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-07-28T16:41:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:24Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681ad64463ffe
    namespace: kserve-ci-e2e-test
    resourceVersion: "20768"
    uid: d25ff0d3-8ad6-4888-8a90-38be0d1c3189
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:26Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.476s (2.476s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:41:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:26Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681adf7e49906
    namespace: kserve-ci-e2e-test
    resourceVersion: "20781"
    uid: 5c8a3451-a8bf-4b0d-a5a6-af2a43e21ab0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:26Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:41:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:26Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681adfa8a8f16
    namespace: kserve-ci-e2e-test
    resourceVersion: "20782"
    uid: 128f62cf-a4fc-41c2-9962-c3fc9bd82bd8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:41:26Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:41:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:41:26Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681adfae94953
    namespace: kserve-ci-e2e-test
    resourceVersion: "20783"
    uid: 6b516105-9562-4427-b0ea-ea193abb8b5b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:42:23Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:41:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:23Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681ae2a4dc644
    namespace: kserve-ci-e2e-test
    resourceVersion: "21175"
    uid: 442116b8-8fe6-4342-b2ff-8833e8da0d8f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:42:43Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:42:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681bfe8d24e4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21339"
    uid: 352bf818-6d18-4b82-89da-1e4c74b84869
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h
    namespace: kserve-ci-e2e-test
    resourceVersion: "20592"
    uid: 50896c6e-d465-4d22-a3ad-21440806e9bf
  kind: Event
  lastTimestamp: "2026-07-28T16:42:43Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:42:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-g972h.18c681bfe8d2d6da
    namespace: kserve-ci-e2e-test
    resourceVersion: "21341"
    uid: 24d84e67-72fb-479d-a9ae-4f1adac73336
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:39:33.034471Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19851"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c681938d9496f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19861"
    uid: 83db182d-1cfe-4ec1-8fd4-45992ddca0c5
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c681939879f1ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "19878"
    uid: 9d1b7afa-aa6e-4420-9da9-f221706a851a
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:34Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19871"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:34Z"
  message: Add eth0 [10.134.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:39:34Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68193cae808f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19898"
    uid: 74747ce0-0771-4fad-b3fa-cc40e6cd621b
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:34Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:34Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68193cc62f002
    namespace: kserve-ci-e2e-test
    resourceVersion: "19900"
    uid: 151865d8-e58c-49af-b71d-5185cd96d327
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:34Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:34Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68193cf11481d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19902"
    uid: 9f1f8874-936f-451b-85f2-c1b75f4742af
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:34Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:34Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68193cf954988
    namespace: kserve-ci-e2e-test
    resourceVersion: "19903"
    uid: 5af2c0ee-0159-4b97-b4af-5f7899c03692
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:38Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68194e4a94c90
    namespace: kserve-ci-e2e-test
    resourceVersion: "19949"
    uid: 0f833a0e-f11a-4bff-9fd7-a0d7007b375d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:38Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68194e799bbd2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19952"
    uid: bcbb3b09-ed08-4ff1-921c-85ff5e20b594
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:38Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68194e808844c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19953"
    uid: 2a52e99e-1a31-46cd-8c98-5ce88e77553f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:38Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68194e811c450
    namespace: kserve-ci-e2e-test
    resourceVersion: "19954"
    uid: a6706b59-e541-43e1-886d-436136b64983
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:38Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68194eab63d1c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19955"
    uid: 54f5e375-984f-41a0-a953-cf1445b467e4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:38Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:39:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:39:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c68194eb2e3914
    namespace: kserve-ci-e2e-test
    resourceVersion: "19956"
    uid: 382605a6-6bfb-4385-8809-ede997f24d61
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:42:45Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:39:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:45Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c681964acd2d9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: 312cedd5-362e-4320-b516-f55657dc1840
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:42:43Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:42:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c681bfe82f056f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21338"
    uid: ae6a9144-9647-4055-a7e8-709a09049c32
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:42:43Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:42:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c681bfe82fb61e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21340"
    uid: 4f336e3b-a78f-47a8-94a2-0e2b021566f2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: b2be2d1b-5fae-40f3-820f-e809c73232e9
  kind: Event
  lastTimestamp: "2026-07-28T16:42:44Z"
  message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp
    10.134.0.30:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:42:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:44Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-w6s7k.18c681c033e37250
    namespace: kserve-ci-e2e-test
    resourceVersion: "21351"
    uid: d0e2b850-68f0-4cc3-921c-b0e2c0423fce
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:33Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-86766c4958
    namespace: kserve-ci-e2e-test
    resourceVersion: "19842"
    uid: bf68e304-9a6a-405c-af3d-23d5ca1fa3d6
  kind: Event
  lastTimestamp: "2026-07-28T16:39:33Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-86766c4958-w6s7k'
  metadata:
    creationTimestamp: "2026-07-28T16:39:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:39:33Z"
    name: isvc-canary-promote-v2-predictor-86766c4958.18c681938cd784e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "19857"
    uid: 4c2de6f9-b43f-4c7c-a271-4ab92e480411
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-86766c4958
    namespace: kserve-ci-e2e-test
    resourceVersion: "20574"
    uid: bf68e304-9a6a-405c-af3d-23d5ca1fa3d6
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-86766c4958-g972h'
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-v2-predictor-86766c4958.18c681a88b69f880
    namespace: kserve-ci-e2e-test
    resourceVersion: "20593"
    uid: 61f259a6-695c-47b9-a8f4-a60ed2cccd14
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:32Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19840"
    uid: 3f145c27-b7e2-4965-8dc0-5768e8e9a3b9
  kind: Event
  lastTimestamp: "2026-07-28T16:39:32Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-86766c4958 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:39:32Z"
    name: isvc-canary-promote-v2-predictor.18c6819389259afb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19844"
    uid: 656ddca6-d2b7-4806-bc6e-044e3b2e7e84
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:41:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20569"
    uid: 3f145c27-b7e2-4965-8dc0-5768e8e9a3b9
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-86766c4958 from
    1 to 2
  metadata:
    creationTimestamp: "2026-07-28T16:41:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote-v2-predictor.18c681a885c52c53
    namespace: kserve-ci-e2e-test
    resourceVersion: "20578"
    uid: 642afc26-2ac1-42b7-a3ef-f537c6ca670c
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "19825"
    uid: 0a9f45c8-7195-4585-993f-731696b68cd6
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote.18c6819388e019c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "20587"
    uid: 927af52c-4390-4b53-a0f5-e24a6db08938
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "19825"
    uid: 0a9f45c8-7195-4585-993f-731696b68cd6
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote.18c6819389687c31
    namespace: kserve-ci-e2e-test
    resourceVersion: "20588"
    uid: 51b0a5a1-a221-49a1-b599-db0e58524b1e
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:39:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "19825"
    uid: 0a9f45c8-7195-4585-993f-731696b68cd6
  kind: Event
  lastTimestamp: "2026-07-28T16:41:03Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:41:03Z"
    name: isvc-canary-promote.18c681938968b7b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20590"
    uid: bbf29cc1-4b51-4518-a550-830b0598606a
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:40:54Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "20501"
    uid: 0a9f45c8-7195-4585-993f-731696b68cd6
  kind: Event
  lastTimestamp: "2026-07-28T16:42:33Z"
  message: InferenceService [isvc-canary-promote] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:40:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:42:33Z"
    name: isvc-canary-promote.18c681a67874551e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21255"
    uid: 19e94662-72b9-4a6e-ac03-1529454eb0e3
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:44:08.643357Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22085"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    to ip-10-0-132-37.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:44:08Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d3b9267aa9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22091"
    uid: 2a00da90-7792-4503-b158-c76f5ef5331b
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:09Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22096"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:09Z"
  message: Add eth0 [10.133.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:44:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:44:09Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d3d26e2ba0
    namespace: kserve-ci-e2e-test
    resourceVersion: "22100"
    uid: 40c66cf6-ba86-4b91-9280-56b917c66976
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:09Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:44:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:09Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d3d3fae161
    namespace: kserve-ci-e2e-test
    resourceVersion: "22102"
    uid: 1c0221e3-b837-4df1-b512-9e6effbea4ee
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:09Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:44:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:09Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d3d680b708
    namespace: kserve-ci-e2e-test
    resourceVersion: "22103"
    uid: 3c29f0f5-33f9-420a-9e1f-69ced41675ae
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:09Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:44:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:09Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d3d6e89fd2
    namespace: kserve-ci-e2e-test
    resourceVersion: "22104"
    uid: b9614b5b-dd47-4ea5-a48c-9d182597a349
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:13Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:44:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:13Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d4becc9f4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22142"
    uid: 094d2972-05d2-4c4e-9bf2-4f0ad46341b2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:13Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:44:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:13Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d4c198cf02
    namespace: kserve-ci-e2e-test
    resourceVersion: "22144"
    uid: b1bbe1b5-07f3-4751-bdd6-607ad206764c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:13Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:44:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:13Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d4c206934d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22145"
    uid: 3043421b-57c5-4238-8237-23e7e6db458e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:13Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:44:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:13Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d4c20ff76a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22146"
    uid: bc20325f-8582-4cb1-a43e-d58545f132e5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:13Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:44:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:13Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d4c51da577
    namespace: kserve-ci-e2e-test
    resourceVersion: "22148"
    uid: 28f2cc2b-ec2c-4759-9452-a7b2b99423f3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:44:13Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:44:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:13Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d4c58b3dc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22149"
    uid: 0c3e1152-2dcd-4c00-88cc-571fed9d4a51
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:45:31Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:44:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:31Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681d536afd763
    namespace: kserve-ci-e2e-test
    resourceVersion: "22718"
    uid: 62eec6e4-eca7-4410-ac11-4462065def6b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:45:28Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:45:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:28Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681e66085ca06
    namespace: kserve-ci-e2e-test
    resourceVersion: "22695"
    uid: da75a945-ef5b-4618-bb0c-f4bca3e7bf9e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:45:28Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:45:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:28Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681e6608650c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22697"
    uid: 99da2055-81de-4c46-a3ce-d3ed786340c8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22087"
    uid: bbce8518-b479-4cdb-a087-e55f6fe77c90
  kind: Event
  lastTimestamp: "2026-07-28T16:45:31Z"
  message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp
    10.133.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:45:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:31Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64.18c681e6e8cb64c7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22717"
    uid: 054f9337-dc0d-4d28-a0e2-4586ecc6a973
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:42:48.616162Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21412"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c1172848d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21422"
    uid: a2fec1ae-511d-4734-9658-94bb068e79a1
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21432"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: Add eth0 [10.134.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c130d6cf27
    namespace: kserve-ci-e2e-test
    resourceVersion: "21447"
    uid: c61455f7-2251-431f-a8d9-f07b6081031d
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c132525d1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21449"
    uid: 65be448b-9cb2-4031-a6e0-ddbd645044f5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c135a78d6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21452"
    uid: 97fab342-e436-4af2-8be5-bf88f9b74ada
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c136186e67
    namespace: kserve-ci-e2e-test
    resourceVersion: "21453"
    uid: 65a4b983-93f7-4df4-993a-484c7c09c339
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c23dd2f9c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "21499"
    uid: 1cffaf6f-7c39-441e-8d59-de992213d8a0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c241cc056e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21505"
    uid: a6790078-86ad-4b01-a559-9cff3f009b8c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c24246078d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21510"
    uid: 0e2d434f-7883-423f-a4c8-1d2913be6bd3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c242748244
    namespace: kserve-ci-e2e-test
    resourceVersion: "21511"
    uid: d00afbf6-d92b-4137-94fd-ec3d87af4854
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c245336a0b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21513"
    uid: 4f43bc6a-ec55-424c-87fd-a0971de67029
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c245a4304b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21516"
    uid: 3a152fcd-0a00-4f35-a5cf-9862cc7c7877
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:45:30Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:42:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:30Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681c279f87d03
    namespace: kserve-ci-e2e-test
    resourceVersion: "22713"
    uid: 414ef694-8dc8-401b-8ac4-32ee0fbe5358
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:45:28Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:45:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:28Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681e66047a569
    namespace: kserve-ci-e2e-test
    resourceVersion: "22694"
    uid: b4951d56-8f42-4bf3-99f2-3e70dd68b372
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:45:28Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:45:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:28Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681e660482d6e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22696"
    uid: 84c94900-956f-4d57-b1d4-f5ca430aa2f2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74
    namespace: kserve-ci-e2e-test
    resourceVersion: "21415"
    uid: 86a9f4c2-13cc-4297-80f3-6e8a89c5ab24
  kind: Event
  lastTimestamp: "2026-07-28T16:45:30Z"
  message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp
    10.134.0.31:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:45:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:30Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74.18c681e6cc80e8f5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22712"
    uid: 7efc7c65-9405-4d4c-a6a8-19c2baba2017
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-5bbb8c8cfc
    namespace: kserve-ci-e2e-test
    resourceVersion: "21396"
    uid: 6867b76b-65d8-4b39-a104-aca987e7dd16
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: 'Created pod: isvc-canary-rollback-predictor-5bbb8c8cfc-qvr74'
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc.18c681c1169f9552
    namespace: kserve-ci-e2e-test
    resourceVersion: "21414"
    uid: 912cd198-e971-405c-8f72-2fa001564bf8
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:08Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-5bbb8c8cfc
    namespace: kserve-ci-e2e-test
    resourceVersion: "22073"
    uid: 6867b76b-65d8-4b39-a104-aca987e7dd16
  kind: Event
  lastTimestamp: "2026-07-28T16:44:08Z"
  message: 'Created pod: isvc-canary-rollback-predictor-5bbb8c8cfc-cxk64'
  metadata:
    creationTimestamp: "2026-07-28T16:44:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc.18c681d3b8adbf7a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22086"
    uid: 80e5f9f4-fbff-4e80-ac43-da85b8f0a516
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21395"
    uid: 2306a3ce-6c4b-4ac9-96b2-2f8cac5f478e
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-5bbb8c8cfc from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-predictor.18c681c1109e91d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21397"
    uid: 57035d16-d609-44d9-80c0-43cc7668acbf
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:08Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22067"
    uid: 2306a3ce-6c4b-4ac9-96b2-2f8cac5f478e
  kind: Event
  lastTimestamp: "2026-07-28T16:44:08Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-5bbb8c8cfc from 1
    to 2
  metadata:
    creationTimestamp: "2026-07-28T16:44:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback-predictor.18c681d3b586a77d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22077"
    uid: a18ca00f-443c-4092-8e32-36b0330abe07
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:42:48.617673Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21413"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c1173f5a01
    namespace: kserve-ci-e2e-test
    resourceVersion: "21423"
    uid: 2c85d2c8-ff33-4afc-aa25-c78643f89f0e
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-rollback-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c11ff44625
    namespace: kserve-ci-e2e-test
    resourceVersion: "21437"
    uid: 90697f5d-43f4-4e17-bec8-c9df1e49d54e
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21433"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: Add eth0 [10.134.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c1550b7677
    namespace: kserve-ci-e2e-test
    resourceVersion: "21464"
    uid: 8f87bf34-e500-4651-8cfe-8f19d0761a3d
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c1565cdef8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21467"
    uid: a79006f7-814e-4188-8404-a40274117003
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c158f5902f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21469"
    uid: cf6272f4-9b93-47a5-94b6-3a24a113878c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:49Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:42:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:49Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c159672e7b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21470"
    uid: 6a64d9f8-c2a7-47f0-8679-6dc1903cfa9d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c23dbe23f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21498"
    uid: 767cff03-f4d8-4fa8-8191-654c0bac7885
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c2416c19c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21504"
    uid: ad523576-4969-4df7-be01-3bdeec029363
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c241f4909b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21506"
    uid: c7c44bf1-c727-49dc-b61d-1679f0678ef0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c241ff6960
    namespace: kserve-ci-e2e-test
    resourceVersion: "21508"
    uid: d7d9fc8e-2f02-4f76-a00a-cc3e718aa77a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c2452207f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21512"
    uid: b9caf3e6-b5a5-40c6-9d7e-1db9dd2cf521
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:42:53Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:42:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:42:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c245956420
    namespace: kserve-ci-e2e-test
    resourceVersion: "21514"
    uid: e459232f-b295-4ff6-8db9-130c8e678fee
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:44:11Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:42:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:11Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681c2b5d01bf1
    namespace: kserve-ci-e2e-test
    resourceVersion: "22129"
    uid: 067dc07c-d377-49f2-937f-4f1dc0e890bc
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:44:08Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:44:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681d3b68a4dee
    namespace: kserve-ci-e2e-test
    resourceVersion: "22082"
    uid: 919d7434-6f36-4490-ac25-237708262f56
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:44:08Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:44:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681d3b68acfac
    namespace: kserve-ci-e2e-test
    resourceVersion: "22083"
    uid: 1f73c1b1-f0c0-48de-b9de-38dd5dcae209
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55
    namespace: kserve-ci-e2e-test
    resourceVersion: "21417"
    uid: 3735a9d1-8d67-48c6-b033-629393b69be5
  kind: Event
  lastTimestamp: "2026-07-28T16:44:11Z"
  message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp
    10.134.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:44:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:44:11Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55.18c681d467f861e9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22127"
    uid: 4fa8a3f2-b908-4448-a80a-9bd81daf73b3
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-v2-predictor-84f48b7656
    namespace: kserve-ci-e2e-test
    resourceVersion: "21405"
    uid: 850864ec-4851-471e-b426-03126f1b7093
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: 'Created pod: isvc-canary-rollback-v2-predictor-84f48b7656-7lq55'
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656.18c681c116ce9cd4
    namespace: kserve-ci-e2e-test
    resourceVersion: "21418"
    uid: 574d8397-0324-4173-b70d-dc8bf995e72c
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21404"
    uid: 26ccae82-a1f7-4291-992b-6caa8c0979bf
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: Scaled up replica set isvc-canary-rollback-v2-predictor-84f48b7656 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback-v2-predictor.18c681c113002cee
    namespace: kserve-ci-e2e-test
    resourceVersion: "21407"
    uid: ae0ae5e4-69cf-4488-96c8-0bb187f480e2
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 20
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "21388"
    uid: 7920b259-9589-428a-8d21-5dc18ba98a52
  kind: Event
  lastTimestamp: "2026-07-28T16:44:08Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback.18c681c11295934f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22097"
    uid: cbb60706-6d30-4935-80d2-b27649bd7d99
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "21388"
    uid: 7920b259-9589-428a-8d21-5dc18ba98a52
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: 'Failed to update status for InferenceService "isvc-canary-rollback": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback.18c681c11329eb7c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21435"
    uid: d11b5364-a218-4fac-b2ca-2254f38e3029
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:42:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "21388"
    uid: 7920b259-9589-428a-8d21-5dc18ba98a52
  kind: Event
  lastTimestamp: "2026-07-28T16:42:48Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:42:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:42:48Z"
    name: isvc-canary-rollback.18c681c1132a2e21
    namespace: kserve-ci-e2e-test
    resourceVersion: "21436"
    uid: ab10b91c-4585-4e6f-b485-0948433763b0
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "22003"
    uid: 7920b259-9589-428a-8d21-5dc18ba98a52
  kind: Event
  lastTimestamp: "2026-07-28T16:45:21Z"
  message: InferenceService [isvc-canary-rollback] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:44:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:45:21Z"
    name: isvc-canary-rollback.18c681d1de9bda46
    namespace: kserve-ci-e2e-test
    resourceVersion: "22627"
    uid: 9e5c6f01-29b4-49ea-89ee-09a4194923f0
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:44:08Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "22090"
    uid: 7920b259-9589-428a-8d21-5dc18ba98a52
  kind: Event
  lastTimestamp: "2026-07-28T16:44:08Z"
  message: 'InferenceService [isvc-canary-rollback] is no longer Ready because of:
    PredictorReady, Ready, Stopped'
  metadata:
    creationTimestamp: "2026-07-28T16:44:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:44:08Z"
    name: isvc-canary-rollback.18c681d3b96a961a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22092"
    uid: f0c9add7-d5eb-42bd-b2c3-2b40c7ea0ae4
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:45:33.988049Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22774"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-86dc9fdb65-7kw22
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:45:33Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e798170328
    namespace: kserve-ci-e2e-test
    resourceVersion: "22782"
    uid: 9fb2a1ef-9cb8-436b-b2a6-4bf737e9c1d0
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22786"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: Add eth0 [10.134.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e7b1a9e266
    namespace: kserve-ci-e2e-test
    resourceVersion: "22806"
    uid: 745e326b-af44-49e1-ab28-eb6548b33d53
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e7b35453f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "22809"
    uid: ce6ed920-eb3c-47c3-a594-7b4d0c462cb9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e7b7c464c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "22813"
    uid: 3e835c7d-6a0c-4f64-b980-c0d2ccc80df3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e7b86c30a1
    namespace: kserve-ci-e2e-test
    resourceVersion: "22814"
    uid: ec4f9475-f807-40e0-acb3-2e4987a1408e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e89391efd7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22854"
    uid: ccfdec8e-2300-4fa3-9355-8ddf6f398ca4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e897027cf7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22858"
    uid: f39c8820-fa5c-4fac-8a56-1cf5c7772b13
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e8977a3725
    namespace: kserve-ci-e2e-test
    resourceVersion: "22862"
    uid: faf830a3-341b-4f29-ad18-71418466efde
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e8978261bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22864"
    uid: bf3d317a-03b6-445f-819b-9cb6e2be125a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e89ae2359c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22867"
    uid: c3f189e4-5628-4609-867d-72aeb338e194
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e89b553460
    namespace: kserve-ci-e2e-test
    resourceVersion: "22868"
    uid: 6956c927-f9a9-4079-a12d-ca0df7a76fac
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:46:36Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:45:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:46:36Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681e90b8893ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "23277"
    uid: 0669e54c-745a-4eab-b41f-c3e77cf573d5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:46:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:46:58Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:46:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:46:58Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681fb610711fa
    namespace: kserve-ci-e2e-test
    resourceVersion: "23469"
    uid: ba6d6cc0-fd82-4192-9be9-2f36985d2901
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:46:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:46:58Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:46:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:46:58Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681fb61083198
    namespace: kserve-ci-e2e-test
    resourceVersion: "23470"
    uid: 8a79b735-da9b-464f-80a4-1ba55f5fdcae
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22
    namespace: kserve-ci-e2e-test
    resourceVersion: "22778"
    uid: e7382f39-a4cd-4cac-bd06-74f83fa67a4f
  kind: Event
  lastTimestamp: "2026-07-28T16:47:01Z"
  message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp
    10.134.0.33:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:47:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:01Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-7kw22.18c681fbe7b32b4c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23500"
    uid: e32f7bfa-9027-4a1f-852c-88b0d9660d4a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:33Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-86dc9fdb65
    namespace: kserve-ci-e2e-test
    resourceVersion: "22758"
    uid: 7b6f0a0d-9819-4ecc-beb4-056693239c59
  kind: Event
  lastTimestamp: "2026-07-28T16:45:33Z"
  message: 'Created pod: isvc-canary-stop-predictor-86dc9fdb65-7kw22'
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:45:33Z"
    name: isvc-canary-stop-predictor-86dc9fdb65.18c681e7975412a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22777"
    uid: c87843e9-28a9-4433-94fb-273613d08df0
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:33Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22757"
    uid: b6fd4436-f6ac-4225-8216-48ea8f07af21
  kind: Event
  lastTimestamp: "2026-07-28T16:45:33Z"
  message: Scaled up replica set isvc-canary-stop-predictor-86dc9fdb65 from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:45:33Z"
    name: isvc-canary-stop-predictor.18c681e7929adb56
    namespace: kserve-ci-e2e-test
    resourceVersion: "22759"
    uid: 340e7e32-be74-4ddc-bfa3-b6d79c545c2b
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:45:34.051712Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22788"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e79be273a1
    namespace: kserve-ci-e2e-test
    resourceVersion: "22795"
    uid: 64adef0e-ac85-448b-93d3-d0d212b35fbd
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22799"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: Add eth0 [10.134.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e7b5717fac
    namespace: kserve-ci-e2e-test
    resourceVersion: "22810"
    uid: 757dbd8a-2945-474f-8cac-fe3e14a890e5
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e7b756ccb2
    namespace: kserve-ci-e2e-test
    resourceVersion: "22812"
    uid: 9c2fec92-88df-41da-9766-7c4f86891f57
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e7ba12f9fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22815"
    uid: 6540e6d0-b9f4-4b6c-bffa-d0abae890280
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e7ba84c1b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22816"
    uid: dc83a1e3-8e3f-46b7-8a41-fa7c8a6eb839
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e8937d4519
    namespace: kserve-ci-e2e-test
    resourceVersion: "22853"
    uid: 799292d1-f8df-4ba9-ad0c-3fea383ae8ae
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e896f6308c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22857"
    uid: 1390655b-a28f-4671-aefb-7f159940dab8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e8976cd723
    namespace: kserve-ci-e2e-test
    resourceVersion: "22859"
    uid: 70ae7236-2794-4ce4-82bf-7830cd70b70d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e897772455
    namespace: kserve-ci-e2e-test
    resourceVersion: "22861"
    uid: 573123e0-9b95-4c07-9f0e-4f916bd8ecfa
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e89ad5a982
    namespace: kserve-ci-e2e-test
    resourceVersion: "22866"
    uid: 276b1ff1-2c40-432c-a826-284a64480fc1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:45:38Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:45:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:45:38Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e89b55357e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22869"
    uid: e89c0f75-eb80-4aef-9d3a-914aa8bc7bc2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:39Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:46:45Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:45:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:46:45Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681e8cf9cafc6
    namespace: kserve-ci-e2e-test
    resourceVersion: "23339"
    uid: 749c58a6-9c39-44cd-94b5-9478f21c448b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:46:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:46:58Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:46:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:46:59Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681fb6324b457
    namespace: kserve-ci-e2e-test
    resourceVersion: "23478"
    uid: 3f2807be-cf74-4164-bc0f-10658fa37e20
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:46:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:46:58Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:46:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:46:59Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681fb6324fa7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23481"
    uid: bdfbdd4b-88e1-4fc8-a276-d26080d08c10
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd
    namespace: kserve-ci-e2e-test
    resourceVersion: "22790"
    uid: 942ab7e5-a937-49ad-b7ec-821f0e1eb850
  kind: Event
  lastTimestamp: "2026-07-28T16:47:00Z"
  message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp
    10.134.0.34:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:47:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:00Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-vjqjd.18c681fbabdb8b54
    namespace: kserve-ci-e2e-test
    resourceVersion: "23493"
    uid: 6b68e1f5-f553-4def-a624-100e6cb32041
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:34Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-v2-predictor-77f778d864
    namespace: kserve-ci-e2e-test
    resourceVersion: "22768"
    uid: f1be7652-4fd1-48ec-8b1f-3355f1334095
  kind: Event
  lastTimestamp: "2026-07-28T16:45:34Z"
  message: 'Created pod: isvc-canary-stop-v2-predictor-77f778d864-vjqjd'
  metadata:
    creationTimestamp: "2026-07-28T16:45:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:45:34Z"
    name: isvc-canary-stop-v2-predictor-77f778d864.18c681e79b00e797
    namespace: kserve-ci-e2e-test
    resourceVersion: "22789"
    uid: 84134665-f1cd-4259-9eec-60bf6a184db5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:33Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22766"
    uid: 504ac5fa-ad76-4e4d-9fca-346ec2c0107e
  kind: Event
  lastTimestamp: "2026-07-28T16:45:33Z"
  message: Scaled up replica set isvc-canary-stop-v2-predictor-77f778d864 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:45:33Z"
    name: isvc-canary-stop-v2-predictor.18c681e795834e6e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22770"
    uid: 0e935f08-1ba6-47fc-9427-fd21a839123e
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 18
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:33Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "22750"
    uid: 3c00d9df-e862-4abb-be6a-2e270d3ed8c8
  kind: Event
  lastTimestamp: "2026-07-28T16:46:59Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:46:59Z"
    name: isvc-canary-stop.18c681e794ce34b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23487"
    uid: 5b9ef9ec-4aa1-4f09-9247-d5bb4b5ea055
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:33Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "22750"
    uid: 3c00d9df-e862-4abb-be6a-2e270d3ed8c8
  kind: Event
  lastTimestamp: "2026-07-28T16:45:33Z"
  message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:45:33Z"
    name: isvc-canary-stop.18c681e79576e5f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22769"
    uid: 84e0ed32-18d3-4ec9-8080-08142920341f
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:45:33Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "22750"
    uid: 3c00d9df-e862-4abb-be6a-2e270d3ed8c8
  kind: Event
  lastTimestamp: "2026-07-28T16:45:33Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:45:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:45:33Z"
    name: isvc-canary-stop.18c681e795772271
    namespace: kserve-ci-e2e-test
    resourceVersion: "22772"
    uid: 440336d7-d1f8-4c78-9dd0-56951cb992e4
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:46:46Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "23352"
    uid: 3c00d9df-e862-4abb-be6a-2e270d3ed8c8
  kind: Event
  lastTimestamp: "2026-07-28T16:46:46Z"
  message: InferenceService [isvc-canary-stop] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:46:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:46:46Z"
    name: isvc-canary-stop.18c681f870939b95
    namespace: kserve-ci-e2e-test
    resourceVersion: "23353"
    uid: eed00110-b234-4166-8b38-0644a5637823
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:46:58Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "23475"
    uid: 3c00d9df-e862-4abb-be6a-2e270d3ed8c8
  kind: Event
  lastTimestamp: "2026-07-28T16:46:58Z"
  message: 'InferenceService [isvc-canary-stop] is no longer Ready because of: PredictorReady,
    Ready'
  metadata:
    creationTimestamp: "2026-07-28T16:46:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:46:58Z"
    name: isvc-canary-stop.18c681fb62aaa885
    namespace: kserve-ci-e2e-test
    resourceVersion: "23477"
    uid: 7cc46045-bc80-4d8c-b3b6-6b1fd3149e7c
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:49:07.902228Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24598"
    uid: c07b7e90-df9d-47b0-ac91-1366923f71d0
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:49:07Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:49:07Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5.18c68219665e62be
    namespace: kserve-ci-e2e-test
    resourceVersion: "24602"
    uid: 97c91349-d94b-443f-b862-80081c148dd2
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:08Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24610"
    uid: c07b7e90-df9d-47b0-ac91-1366923f71d0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:08Z"
  message: Add eth0 [10.134.0.37/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:49:08Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5.18c682197f941d9f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24618"
    uid: c4ba0204-4df4-4c4c-be2a-09312ca38663
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24600"
    uid: c07b7e90-df9d-47b0-ac91-1366923f71d0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:15Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:15Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5.18c68219810c9dd1
    namespace: kserve-ci-e2e-test
    resourceVersion: "24674"
    uid: e8ed7793-7091-42bc-a295-5c735f50c761
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24600"
    uid: c07b7e90-df9d-47b0-ac91-1366923f71d0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:15Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:15Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5.18c682198362b235
    namespace: kserve-ci-e2e-test
    resourceVersion: "24676"
    uid: df38f5f1-4a6d-455d-865e-abe432510c07
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24600"
    uid: c07b7e90-df9d-47b0-ac91-1366923f71d0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:15Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:15Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5.18c6821983d2f896
    namespace: kserve-ci-e2e-test
    resourceVersion: "24677"
    uid: 05c63dc3-8904-4486-bc64-3dda233446c1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24600"
    uid: c07b7e90-df9d-47b0-ac91-1366923f71d0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:17Z"
  message: Stopping container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:49:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:17Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5.18c6821bb8e08006
    namespace: kserve-ci-e2e-test
    resourceVersion: "24718"
    uid: 0d331841-6a7c-499e-bedb-950bfc8de4ab
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-init-fail-1ebd71-predictor-77998b7f76
    namespace: kserve-ci-e2e-test
    resourceVersion: "24591"
    uid: d8c44274-0db0-43b7-ac88-1f2cca2d4db5
  kind: Event
  lastTimestamp: "2026-07-28T16:49:07Z"
  message: 'Created pod: isvc-init-fail-1ebd71-predictor-77998b7f76-d6bp5'
  metadata:
    creationTimestamp: "2026-07-28T16:49:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:49:07Z"
    name: isvc-init-fail-1ebd71-predictor-77998b7f76.18c6821965b75a57
    namespace: kserve-ci-e2e-test
    resourceVersion: "24599"
    uid: e0909574-f403-4a10-966f-a191cb43284a
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-init-fail-1ebd71-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24589"
    uid: a5a767c5-2eb2-40b2-b4c9-1768c3f54aed
  kind: Event
  lastTimestamp: "2026-07-28T16:49:07Z"
  message: Scaled up replica set isvc-init-fail-1ebd71-predictor-77998b7f76 from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T16:49:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:49:07Z"
    name: isvc-init-fail-1ebd71-predictor.18c6821962ea0bd2
    namespace: kserve-ci-e2e-test
    resourceVersion: "24592"
    uid: c042b33f-1c2b-4449-b4df-4cd2eb7c958e
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-1ebd71
    namespace: kserve-ci-e2e-test
    resourceVersion: "24582"
    uid: f698f789-5531-4de8-97b9-5a5eaf29bcca
  kind: Event
  lastTimestamp: "2026-07-28T16:49:15Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:49:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:49:15Z"
    name: isvc-init-fail-1ebd71.18c6821962eec83a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24681"
    uid: 1302fbca-05e2-4e63-b285-a5fa6394beca
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-1ebd71
    namespace: kserve-ci-e2e-test
    resourceVersion: "24582"
    uid: f698f789-5531-4de8-97b9-5a5eaf29bcca
  kind: Event
  lastTimestamp: "2026-07-28T16:49:07Z"
  message: 'Failed to update status for InferenceService "isvc-init-fail-1ebd71":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-1ebd71":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:49:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:49:07Z"
    name: isvc-init-fail-1ebd71.18c6821963868e7e
    namespace: kserve-ci-e2e-test
    resourceVersion: "24595"
    uid: 6ca3f086-c0bb-4623-a37c-0c6afb1207e4
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-1ebd71
    namespace: kserve-ci-e2e-test
    resourceVersion: "24582"
    uid: f698f789-5531-4de8-97b9-5a5eaf29bcca
  kind: Event
  lastTimestamp: "2026-07-28T16:49:07Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-init-fail-1ebd71": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:49:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:49:07Z"
    name: isvc-init-fail-1ebd71.18c682196386d5fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "24596"
    uid: 8062998c-2547-4ad0-9262-7430c519ff72
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:33:52.038548Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17054"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:33:52Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:33:52Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6814428a3b9b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17059"
    uid: b5d0514e-1ad1-4cef-9362-4e78cbf9a530
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:52Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-9ed13-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:33:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:52Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6814431254fe9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17067"
    uid: b2ad9891-e490-4593-9c04-f2d8c7351112
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:53Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17063"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:53Z"
  message: Add eth0 [10.134.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:33:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:33:53Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6814466147deb
    namespace: kserve-ci-e2e-test
    resourceVersion: "17081"
    uid: 76b18442-ef66-4936-9f02-574e9394f7e2
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:53Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:33:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:53Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6814467ab3c27
    namespace: kserve-ci-e2e-test
    resourceVersion: "17083"
    uid: 75da8cee-a072-4493-b00f-bf261848c9fd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:53Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:33:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:53Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c681446b3bd4d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17084"
    uid: 5ca32b49-95c5-4b21-b65e-9b834281b1b6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:53Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:33:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:53Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c681446baa9109
    namespace: kserve-ci-e2e-test
    resourceVersion: "17085"
    uid: 16a87970-d26d-4f04-92c5-848732bb5092
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c681456aaddbad
    namespace: kserve-ci-e2e-test
    resourceVersion: "17124"
    uid: e0119056-91dc-47bd-bfbe-37c3a285901d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c681456d6e13fa
    namespace: kserve-ci-e2e-test
    resourceVersion: "17127"
    uid: cbe98e52-94bb-4f2c-955c-80d2c28605f5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c681456dd5ac76
    namespace: kserve-ci-e2e-test
    resourceVersion: "17128"
    uid: b1b4fe68-77d4-4eca-b172-63a2f86e8d4a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c681456de057d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17129"
    uid: 9b444420-2276-46e8-9294-580e747b1aa9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c68145709caa2b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17132"
    uid: b466c3eb-681a-45b1-a0ce-08b792918d28
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6814571229819
    namespace: kserve-ci-e2e-test
    resourceVersion: "17133"
    uid: 04fb93e1-3394-41f6-aac3-2d7df532bb2e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: Container image "quay.io/opendatahub/kserve-agent@sha256:efaf0bd4be5e59e0fd5df072416f565ae616956e4a23f814ea31d63b5efa6d4b"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c68145712cbcd7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17135"
    uid: 0701bbdc-8c94-4907-920a-999a102ad912
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c68145741ef8e7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17137"
    uid: 48f0443d-66e5-442d-8e4e-c7c3641ef4a6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:57Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-07-28T16:33:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:57Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c68145749095e7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17138"
    uid: 163b9b00-e253-48e5-86a4-670e6b54690f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:35Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:33:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:35Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c68145a6b56663
    namespace: kserve-ci-e2e-test
    resourceVersion: "17962"
    uid: b8ca262e-2913-4420-94b9-a3f7586d0031
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:35Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T16:33:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:35Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c68145e27de963
    namespace: kserve-ci-e2e-test
    resourceVersion: "17963"
    uid: db64f8a2-c408-415f-bbdc-2d154b80657a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6815a577f27d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17831"
    uid: faef41fb-6c8f-42fe-abf7-14fc60f09cac
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6815a577f7459
    namespace: kserve-ci-e2e-test
    resourceVersion: "17832"
    uid: 6b6d3fb2-b4de-4c40-b766-1cfdfc308958
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6815a578030b6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17834"
    uid: 7af4cffb-c274-411c-af4d-257dd6c46d78
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg
    namespace: kserve-ci-e2e-test
    resourceVersion: "17056"
    uid: ae69928a-1e66-435e-b05f-7ca27835572e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:40Z"
  message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp
    10.134.0.25:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:35:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:40Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg.18c6815b12b73962
    namespace: kserve-ci-e2e-test
    resourceVersion: "18003"
    uid: 69644488-fb3c-4282-a8a6-e232d4dabb4f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:52Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-9ed13-predictor-7c557d587
    namespace: kserve-ci-e2e-test
    resourceVersion: "17043"
    uid: 9091aaab-616d-46db-b4de-00c8f94af572
  kind: Event
  lastTimestamp: "2026-07-28T16:33:52Z"
  message: 'Created pod: isvc-logger-raw-9ed13-predictor-7c557d587-k8dsg'
  metadata:
    creationTimestamp: "2026-07-28T16:33:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:52Z"
    name: isvc-logger-raw-9ed13-predictor-7c557d587.18c68144283192c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17057"
    uid: 6b56b2e5-1d2b-4894-aa62-08266836b58e
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17041"
    uid: 557866b2-343d-4265-9630-9fe13b36160a
  kind: Event
  lastTimestamp: "2026-07-28T16:33:51Z"
  message: Scaled up replica set isvc-logger-raw-9ed13-predictor-7c557d587 from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T16:33:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:51Z"
    name: isvc-logger-raw-9ed13-predictor.18c6814424221209
    namespace: kserve-ci-e2e-test
    resourceVersion: "17045"
    uid: fa5dedf8-f8f4-429c-9976-35af52cb4164
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:34:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17042"
    uid: f5b2f017-97a7-4478-8af5-ffac0d723458
  kind: Event
  lastTimestamp: "2026-07-28T16:34:21Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:34:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:34:22Z"
    name: isvc-logger-raw-9ed13-predictor.18c68147a29da8e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17325"
    uid: a5210ba1-ef8d-4f41-a3cc-8f6e9c938bf3
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:34:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17042"
    uid: f5b2f017-97a7-4478-8af5-ffac0d723458
  kind: Event
  lastTimestamp: "2026-07-28T16:34:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:34:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:34:22Z"
    name: isvc-logger-raw-9ed13-predictor.18c68147a311cb7b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17326"
    uid: fdd69c33-b261-4bd4-91fe-ac24428f77a1
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:34:37Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17210"
    uid: f5b2f017-97a7-4478-8af5-ffac0d723458
  kind: Event
  lastTimestamp: "2026-07-28T16:35:22Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:34:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:35:22Z"
    name: isvc-logger-raw-9ed13-predictor.18c6814ea13b2c0e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17763"
    uid: b6b60c8d-994c-4f3c-91a0-8b828b9e6d5d
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:34:37Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17210"
    uid: f5b2f017-97a7-4478-8af5-ffac0d723458
  kind: Event
  lastTimestamp: "2026-07-28T16:35:22Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:34:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:35:22Z"
    name: isvc-logger-raw-9ed13-predictor.18c6814ea1aa3176
    namespace: kserve-ci-e2e-test
    resourceVersion: "17764"
    uid: 19e1c2b7-b242-4fb2-a1ca-717777e80428
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-9ed13
    namespace: kserve-ci-e2e-test
    resourceVersion: "17034"
    uid: 2abf9cfc-5def-4067-8e53-f3f8d89a3f04
  kind: Event
  lastTimestamp: "2026-07-28T16:35:15Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:33:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:35:15Z"
    name: isvc-logger-raw-9ed13.18c6814424292d51
    namespace: kserve-ci-e2e-test
    resourceVersion: "17723"
    uid: 19f31611-a87b-4f8a-b832-0d3a31115212
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-9ed13
    namespace: kserve-ci-e2e-test
    resourceVersion: "17034"
    uid: 2abf9cfc-5def-4067-8e53-f3f8d89a3f04
  kind: Event
  lastTimestamp: "2026-07-28T16:33:51Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-9ed13":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-9ed13":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:33:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:51Z"
    name: isvc-logger-raw-9ed13.18c6814424c11fd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17048"
    uid: 9de3467c-52d5-45fc-88a7-3c6f1961e72d
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:15Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-9ed13
    namespace: kserve-ci-e2e-test
    resourceVersion: "17719"
    uid: 2abf9cfc-5def-4067-8e53-f3f8d89a3f04
  kind: Event
  lastTimestamp: "2026-07-28T16:35:15Z"
  message: InferenceService [isvc-logger-raw-9ed13] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:35:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:35:15Z"
    name: isvc-logger-raw-9ed13.18c681579b5eab12
    namespace: kserve-ci-e2e-test
    resourceVersion: "17720"
    uid: b6eb4560-ed01-4633-8b24-c2e74630cd6d
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:47:09.291118Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23590"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:47:09Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681fdc8988b3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23595"
    uid: 6ab3b40c-081f-4412-b284-132911e34fc7
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:09Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:09Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-2a8c3a-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:09Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681fdd38caedf
    namespace: kserve-ci-e2e-test
    resourceVersion: "23605"
    uid: 7084d408-b1b1-49b7-b692-3e97c64bd446
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:10Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23599"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:10Z"
  message: Add eth0 [10.134.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:47:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:47:10Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681fe06c7b7b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23613"
    uid: 68d71ae9-fb05-4758-82c9-b83e30344595
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:10Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:47:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:10Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681fe0856e8c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "23615"
    uid: e00261d8-e906-45bd-ad94-3982a52f9b4f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:10Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:47:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:10Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681fe0aebd56c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23618"
    uid: 8daa27fb-a4e0-4c1d-a9fc-37a2784a1bf8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:10Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:47:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:10Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681fe0b5b10af
    namespace: kserve-ci-e2e-test
    resourceVersion: "23619"
    uid: c9a801b8-f626-48d5-aebe-3015c4d24d70
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:14Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:47:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:14Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff059146ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "23653"
    uid: a2bd9cec-220b-41cc-b464-9816ff76ae9f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:14Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:47:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:14Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff08421140
    namespace: kserve-ci-e2e-test
    resourceVersion: "23656"
    uid: e2ba42a6-ce20-4bc2-a040-a30128a19097
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:14Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:47:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:14Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff08b52f8d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23657"
    uid: 4155d3a2-67b5-4f03-9d21-1d57a380e878
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:14Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:47:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:14Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff08be17ab
    namespace: kserve-ci-e2e-test
    resourceVersion: "23658"
    uid: 16c07db1-3838-4190-bddf-f4aeab4caaa7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:14Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:47:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:14Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff0b9ad7ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "23661"
    uid: fe370990-9e9f-4e0f-a4b1-04d42640f8bc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:47:14Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:47:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:47:14Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff0c0c7aee
    namespace: kserve-ci-e2e-test
    resourceVersion: "23662"
    uid: 0ba45cbd-5679-4532-ae37-108320483bb1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:48:12Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:47:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:48:12Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c681ff419009a1
    namespace: kserve-ci-e2e-test
    resourceVersion: "24079"
    uid: 993bf000-aaca-4efc-8875-14e93eb14b28
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:49:02Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:49:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:02Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c682182fa414be
    namespace: kserve-ci-e2e-test
    resourceVersion: "24531"
    uid: 15177a18-1ebf-486e-9d2f-b9073925ae6a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23593"
    uid: 1e94e79f-271c-42f2-a910-eb4f95df0ba3
  kind: Event
  lastTimestamp: "2026-07-28T16:49:02Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:49:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:02Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5.18c682182fa559b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "24532"
    uid: 36635c3a-43bd-452f-9178-f5ccaeef70c4
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:09Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-primary-2a8c3a-predictor-55d8dc79df
    namespace: kserve-ci-e2e-test
    resourceVersion: "23581"
    uid: 957ee989-167d-4394-8d67-12493e472feb
  kind: Event
  lastTimestamp: "2026-07-28T16:47:09Z"
  message: 'Created pod: isvc-primary-2a8c3a-predictor-55d8dc79df-zfbc5'
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:47:09Z"
    name: isvc-primary-2a8c3a-predictor-55d8dc79df.18c681fdc829d475
    namespace: kserve-ci-e2e-test
    resourceVersion: "23592"
    uid: 9b164b19-f588-4d5e-a85d-f05ee4c4856b
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:09Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-primary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23578"
    uid: 77b94757-19d6-4706-9e47-4bf9fece8c80
  kind: Event
  lastTimestamp: "2026-07-28T16:47:09Z"
  message: Scaled up replica set isvc-primary-2a8c3a-predictor-55d8dc79df from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:47:09Z"
    name: isvc-primary-2a8c3a-predictor.18c681fdc3161142
    namespace: kserve-ci-e2e-test
    resourceVersion: "23582"
    uid: f013b05a-80e2-407d-910e-1dbf250da264
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:24Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23580"
    uid: 2fc3e5ba-e2b9-4993-adcd-a50bf097701d
  kind: Event
  lastTimestamp: "2026-07-28T16:47:39Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:47:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:47:39Z"
    name: isvc-primary-2a8c3a-predictor.18c6820142149dad
    namespace: kserve-ci-e2e-test
    resourceVersion: "23838"
    uid: f1b52878-2ed4-4271-bad2-8b50743eb6fc
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:24Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23580"
    uid: 2fc3e5ba-e2b9-4993-adcd-a50bf097701d
  kind: Event
  lastTimestamp: "2026-07-28T16:47:39Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:47:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:47:39Z"
    name: isvc-primary-2a8c3a-predictor.18c68201428c7660
    namespace: kserve-ci-e2e-test
    resourceVersion: "23839"
    uid: f4f19982-7ad1-49a6-9249-28693b1ce0cf
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:54Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23738"
    uid: 2fc3e5ba-e2b9-4993-adcd-a50bf097701d
  kind: Event
  lastTimestamp: "2026-07-28T16:48:39Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:47:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:48:39Z"
    name: isvc-primary-2a8c3a-predictor.18c6820840cc0abf
    namespace: kserve-ci-e2e-test
    resourceVersion: "24269"
    uid: b316c24e-5ba2-4a7a-a3c0-218cb091e17e
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:54Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23738"
    uid: 2fc3e5ba-e2b9-4993-adcd-a50bf097701d
  kind: Event
  lastTimestamp: "2026-07-28T16:48:39Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:47:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:48:39Z"
    name: isvc-primary-2a8c3a-predictor.18c68208414884d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "24270"
    uid: f2f9bca6-a99a-4470-98c1-ec01dc24b4d2
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:09Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23571"
    uid: 79e02a40-8d88-4c7c-b715-5414d750e656
  kind: Event
  lastTimestamp: "2026-07-28T16:48:22Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:48:22Z"
    name: isvc-primary-2a8c3a.18c681fdc3276c3b
    namespace: kserve-ci-e2e-test
    resourceVersion: "24149"
    uid: a8ee50eb-c9eb-46fa-84a0-894d7635a139
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:09Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23571"
    uid: 79e02a40-8d88-4c7c-b715-5414d750e656
  kind: Event
  lastTimestamp: "2026-07-28T16:47:09Z"
  message: 'Failed to update status for InferenceService "isvc-primary-2a8c3a": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-2a8c3a":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:47:09Z"
    name: isvc-primary-2a8c3a.18c681fdc3c4fa90
    namespace: kserve-ci-e2e-test
    resourceVersion: "23601"
    uid: 33f3d0cc-e8fd-41ec-8c19-66acc4e70202
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:47:09Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23571"
    uid: 79e02a40-8d88-4c7c-b715-5414d750e656
  kind: Event
  lastTimestamp: "2026-07-28T16:47:09Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-primary-2a8c3a": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:47:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:47:09Z"
    name: isvc-primary-2a8c3a.18c681fdc3c5532f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23602"
    uid: 8ef8f4e3-6d2e-452c-ab5f-3b795acb6689
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:22Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24146"
    uid: 79e02a40-8d88-4c7c-b715-5414d750e656
  kind: Event
  lastTimestamp: "2026-07-28T16:48:22Z"
  message: InferenceService [isvc-primary-2a8c3a] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:48:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:48:22Z"
    name: isvc-primary-2a8c3a.18c6820ee1d4ab9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24148"
    uid: f4c1c7b3-7cee-4024-8152-00ce1d69fd3c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:28:36.121144Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14119"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fa9a7eab1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14123"
    uid: 58b1295d-c855-4cec-85fa-2bad92d2c80f
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14129"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: Add eth0 [10.134.0.19/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fab3c8ad6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "14142"
    uid: f263faf8-b8af-4bcd-b2f9-357d3a191a37
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fab54df455
    namespace: kserve-ci-e2e-test
    resourceVersion: "14146"
    uid: 1344b4aa-6964-4891-aaac-0200b3bb25ee
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fab82a34db
    namespace: kserve-ci-e2e-test
    resourceVersion: "14147"
    uid: 3ef39fe0-cf6d-45b0-ab76-703bbeae79d9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fab89d1bfc
    namespace: kserve-ci-e2e-test
    resourceVersion: "14148"
    uid: ad85ac5b-5ba8-4e05-b50e-e25c107c235a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:41Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
  metadata:
    creationTimestamp: "2026-07-28T16:28:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:41Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fbc2f8a0b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14187"
    uid: fea22397-81e3-4184-b91c-e7a5c60effe8
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:54Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    in 13.245s (13.245s including waiting). Image size: 1555831221 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:28:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:54Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fed87d8aee
    namespace: kserve-ci-e2e-test
    resourceVersion: "14285"
    uid: 01c055ef-a82d-4d84-8087-d3f8c939d981
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:54Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:28:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:54Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fedb67349b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14287"
    uid: 3ebb6260-8e59-4425-b366-0b0cb0e92bda
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:54Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:28:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:54Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fedbd980c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "14288"
    uid: 52f27813-73c9-4add-a6f2-a4a07e14b198
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:54Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-07-28T16:28:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:54Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680fedbe5648b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14289"
    uid: 5cf1b0e2-637f-4171-953a-416d709039e4
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:56Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.151s (2.151s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:28:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:56Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680ff5c26c1d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14303"
    uid: bb2975e2-4110-4926-8dff-a5b8bc552bd9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:56Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:28:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:56Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680ff5f04a528
    namespace: kserve-ci-e2e-test
    resourceVersion: "14304"
    uid: 0d3c0d7c-3ba9-4468-b55b-29fd7bbe54b8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:56Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:28:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:56Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680ff5f763627
    namespace: kserve-ci-e2e-test
    resourceVersion: "14305"
    uid: b1cd0e98-ab53-4832-aa3e-e043e444f155
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:56Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:efaf0bd4be5e59e0fd5df072416f565ae616956e4a23f814ea31d63b5efa6d4b"
  metadata:
    creationTimestamp: "2026-07-28T16:28:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:56Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c680ff5f836eb5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14306"
    uid: ab9a1fcb-3979-43c3-82da-90f972ff55fe
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:59Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:efaf0bd4be5e59e0fd5df072416f565ae616956e4a23f814ea31d63b5efa6d4b"
    in 2.717s (2.717s including waiting). Image size: 240168571 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:28:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:59Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c68100017ba54d
    namespace: kserve-ci-e2e-test
    resourceVersion: "14325"
    uid: 56683565-6cef-473e-91b6-751a2499b36b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:59Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-07-28T16:28:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:59Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c6810004864a5c
    namespace: kserve-ci-e2e-test
    resourceVersion: "14326"
    uid: c34b7d80-b7c0-4b95-87bc-434adbfff48a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:28:59Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-07-28T16:28:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:28:59Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c6810004f49637
    namespace: kserve-ci-e2e-test
    resourceVersion: "14327"
    uid: d2c2f9f1-61e1-4eaa-8831-830620fe2cd3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T16:29:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:30:23Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:29:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:23Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c6810035ad3400
    namespace: kserve-ci-e2e-test
    resourceVersion: "15072"
    uid: 58f79475-d9e1-4ee7-bd86-588f285ca88c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:29:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:30:23Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T16:29:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:23Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c68100717a7908
    namespace: kserve-ci-e2e-test
    resourceVersion: "15073"
    uid: 2c0da0fa-f227-4242-89f8-1d3054df044e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c68110c901f6e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14902"
    uid: 5178f06d-ce3d-4d41-a9f1-2444eefd8b27
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c68110c902ffe9
    namespace: kserve-ci-e2e-test
    resourceVersion: "14904"
    uid: 1b239612-84cd-402d-8375-14683d8cd22c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c68110c9035a40
    namespace: kserve-ci-e2e-test
    resourceVersion: "14906"
    uid: f6aa03ae-cabc-44b4-8449-9b3293e6613e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t
    namespace: kserve-ci-e2e-test
    resourceVersion: "14121"
    uid: 4344bdb3-0c32-4e81-b860-5c8a45f8b2d8
  kind: Event
  lastTimestamp: "2026-07-28T16:30:31Z"
  message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp
    10.134.0.19:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:31Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t.18c68111e7cc9863
    namespace: kserve-ci-e2e-test
    resourceVersion: "15140"
    uid: 07abcbf9-3f46-4ff8-8fe5-3f262ae3b13e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "14107"
    uid: 0eee67cc-c683-47ad-8ce7-101f96a50be1
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: 'Created pod: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d-ksg5t'
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d.18c680fa9a0eef8a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14120"
    uid: edabf883-0e0b-4ac7-b6ac-373b1aa9e417
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:35Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-raw-sklearn-batcher-965b2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14106"
    uid: 066f5287-d6f4-4d26-b3c6-46c032015232
  kind: Event
  lastTimestamp: "2026-07-28T16:28:35Z"
  message: Scaled up replica set isvc-raw-sklearn-batcher-965b2-predictor-5b7d68bf6d
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:28:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:28:35Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor.18c680fa92626b51
    namespace: kserve-ci-e2e-test
    resourceVersion: "14109"
    uid: 0308290a-29dc-4761-b091-16ce754706af
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:51Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-965b2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14115"
    uid: 8c3ec76c-da5a-476f-b9b6-d01ae9756ec2
  kind: Event
  lastTimestamp: "2026-07-28T16:29:21Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:28:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:29:21Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor.18c680fe182bc170
    namespace: kserve-ci-e2e-test
    resourceVersion: "14486"
    uid: 5ae2bd33-315c-4141-bad0-05627542437c
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:51Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-965b2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14115"
    uid: 8c3ec76c-da5a-476f-b9b6-d01ae9756ec2
  kind: Event
  lastTimestamp: "2026-07-28T16:29:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:28:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:29:21Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor.18c680fe189b4b14
    namespace: kserve-ci-e2e-test
    resourceVersion: "14487"
    uid: 2e12c734-b6ce-41c5-bb6a-1bceb61cb6ac
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:29:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-965b2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14261"
    uid: 8c3ec76c-da5a-476f-b9b6-d01ae9756ec2
  kind: Event
  lastTimestamp: "2026-07-28T16:30:06Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:29:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:06Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor.18c6810895bdbd49
    namespace: kserve-ci-e2e-test
    resourceVersion: "14821"
    uid: 7e11acca-5f3f-40e0-8b78-4cff8fd99fea
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:29:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-965b2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14261"
    uid: 8c3ec76c-da5a-476f-b9b6-d01ae9756ec2
  kind: Event
  lastTimestamp: "2026-07-28T16:30:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:29:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:06Z"
    name: isvc-raw-sklearn-batcher-965b2-predictor.18c68108962da4ac
    namespace: kserve-ci-e2e-test
    resourceVersion: "14822"
    uid: aaff0d93-0b95-4d82-accd-8dd2f435c720
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 13
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-965b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14097"
    uid: 8784ef61-2c2c-4ce3-aef8-673c944134dd
  kind: Event
  lastTimestamp: "2026-07-28T16:30:03Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:30:03Z"
    name: isvc-raw-sklearn-batcher-965b2.18c680fa98888412
    namespace: kserve-ci-e2e-test
    resourceVersion: "14804"
    uid: 20d2cc4d-0ad7-4d17-8ebd-cd9567229c34
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-965b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14097"
    uid: 8784ef61-2c2c-4ce3-aef8-673c944134dd
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-965b2":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-965b2":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2.18c680fa98ea8d33
    namespace: kserve-ci-e2e-test
    resourceVersion: "14132"
    uid: e7609ed3-b79d-4e3f-9d29-50ad8936d83c
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:28:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-965b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14097"
    uid: 8784ef61-2c2c-4ce3-aef8-673c944134dd
  kind: Event
  lastTimestamp: "2026-07-28T16:28:36Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-965b2": the object
    has been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:28:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:28:36Z"
    name: isvc-raw-sklearn-batcher-965b2.18c680fa98eae81a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14133"
    uid: 0c57d6ea-8ac7-4f84-ace2-52e8f3c96d42
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-965b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "14801"
    uid: 8784ef61-2c2c-4ce3-aef8-673c944134dd
  kind: Event
  lastTimestamp: "2026-07-28T16:30:03Z"
  message: InferenceService [isvc-raw-sklearn-batcher-965b2] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:30:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:30:03Z"
    name: isvc-raw-sklearn-batcher-965b2.18c6810ee95b99c5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14802"
    uid: ea83877a-7ed9-4ae5-a750-25a24d1b669a
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:48:44.543329Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24332"
    uid: df8e4d27-5261-4b7b-be82-9a84d5065c6b
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:48:44Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc.18c68213f611f3f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "24338"
    uid: 97f74b2a-ed89-4be7-bbc1-e0effb852080
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24343"
    uid: df8e4d27-5261-4b7b-be82-9a84d5065c6b
  kind: Event
  lastTimestamp: "2026-07-28T16:48:44Z"
  message: Add eth0 [10.134.0.36/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:48:44Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc.18c682140f4f23ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "24351"
    uid: 519267ac-c045-462e-b6f4-d1312f8ef554
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24334"
    uid: df8e4d27-5261-4b7b-be82-9a84d5065c6b
  kind: Event
  lastTimestamp: "2026-07-28T16:48:51Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:48:51Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc.18c6821410f2c160
    namespace: kserve-ci-e2e-test
    resourceVersion: "24408"
    uid: 6851fd7e-473b-43f9-9918-3774f412c9da
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24334"
    uid: df8e4d27-5261-4b7b-be82-9a84d5065c6b
  kind: Event
  lastTimestamp: "2026-07-28T16:48:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:48:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:48:52Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc.18c6821413edda09
    namespace: kserve-ci-e2e-test
    resourceVersion: "24410"
    uid: ce43f096-d8cb-4ab6-bfdf-2aa564c008fc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24334"
    uid: df8e4d27-5261-4b7b-be82-9a84d5065c6b
  kind: Event
  lastTimestamp: "2026-07-28T16:48:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:48:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:48:52Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc.18c68214146fc446
    namespace: kserve-ci-e2e-test
    resourceVersion: "24412"
    uid: d0bf1d78-2d0a-4638-8818-2760108f391d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24334"
    uid: df8e4d27-5261-4b7b-be82-9a84d5065c6b
  kind: Event
  lastTimestamp: "2026-07-28T16:48:55Z"
  message: Back-off restarting failed container storage-initializer in pod isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc_kserve-ci-e2e-test(df8e4d27-5261-4b7b-be82-9a84d5065c6b)
  metadata:
    creationTimestamp: "2026-07-28T16:48:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:48:55Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc.18c6821665d36e11
    namespace: kserve-ci-e2e-test
    resourceVersion: "24435"
    uid: 24541683-1d5e-4b4f-b585-1d51fd9aeba3
  reason: BackOff
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "24323"
    uid: b26c958f-ab40-4ff7-84b5-190c638fef6a
  kind: Event
  lastTimestamp: "2026-07-28T16:48:44Z"
  message: 'Created pod: isvc-secondary-2a8c3a-predictor-5bc45d97d6-9zqnc'
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:48:44Z"
    name: isvc-secondary-2a8c3a-predictor-5bc45d97d6.18c68213f5a675ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "24333"
    uid: ff5061fe-1c57-4683-8c18-8599374e8919
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-secondary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24321"
    uid: 60cbdd14-de45-46c3-b905-00054bf699cf
  kind: Event
  lastTimestamp: "2026-07-28T16:48:44Z"
  message: Scaled up replica set isvc-secondary-2a8c3a-predictor-5bc45d97d6 from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:48:44Z"
    name: isvc-secondary-2a8c3a-predictor.18c68213f0c541c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "24324"
    uid: d3790738-9527-4c6b-bfd4-bd08288eedb5
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:59Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24322"
    uid: abe31137-d474-4e2e-9e56-4777fc77d8b6
  kind: Event
  lastTimestamp: "2026-07-28T16:48:59Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:48:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:48:59Z"
    name: isvc-secondary-2a8c3a-predictor.18c682176f5e3106
    namespace: kserve-ci-e2e-test
    resourceVersion: "24471"
    uid: 1a30cf38-2d55-4f90-8aa4-552d1a0abef3
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:59Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-2a8c3a-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24322"
    uid: abe31137-d474-4e2e-9e56-4777fc77d8b6
  kind: Event
  lastTimestamp: "2026-07-28T16:48:59Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:48:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:48:59Z"
    name: isvc-secondary-2a8c3a-predictor.18c682176fc2c50f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24473"
    uid: d23bdf38-d532-475f-8af3-82219af0f7fc
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24314"
    uid: ab08d49b-2a09-4457-87e0-4186659ca521
  kind: Event
  lastTimestamp: "2026-07-28T16:48:55Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:48:55Z"
    name: isvc-secondary-2a8c3a.18c68213f0ca8a6a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24441"
    uid: 36a4d844-3e4e-4ceb-aa4c-2b757f35da7e
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24314"
    uid: ab08d49b-2a09-4457-87e0-4186659ca521
  kind: Event
  lastTimestamp: "2026-07-28T16:48:44Z"
  message: 'Failed to update status for InferenceService "isvc-secondary-2a8c3a":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-2a8c3a":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:48:44Z"
    name: isvc-secondary-2a8c3a.18c68213f15ac04a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24344"
    uid: 165e5008-c149-4cca-b98a-2e02da2e6761
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:48:44Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-2a8c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24314"
    uid: ab08d49b-2a09-4457-87e0-4186659ca521
  kind: Event
  lastTimestamp: "2026-07-28T16:48:44Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-secondary-2a8c3a": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:48:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:48:44Z"
    name: isvc-secondary-2a8c3a.18c68213f15af865
    namespace: kserve-ci-e2e-test
    resourceVersion: "24345"
    uid: 5304c4e6-14b7-41af-a45f-d4c96d529835
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:30:11.390977Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14897"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68110c904f86e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14903"
    uid: 70f7f951-12ac-4ee1-a612-212967b6e399
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14917"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Add eth0 [10.134.0.20/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68110e2981d95
    namespace: kserve-ci-e2e-test
    resourceVersion: "14950"
    uid: fc9cb33a-0d72-4a25-b239-b9fb1dbd80a1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68110e419139d
    namespace: kserve-ci-e2e-test
    resourceVersion: "14952"
    uid: 431cc546-90ba-4cc9-8cd7-61f733be9551
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68110e7543d31
    namespace: kserve-ci-e2e-test
    resourceVersion: "14953"
    uid: cdf47d1e-0674-4407-b45d-6d19afc43687
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68110e7b9f99e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14954"
    uid: bf85d647-ce82-405e-8cb7-bbc395798e5a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68111f9cb1737
    namespace: kserve-ci-e2e-test
    resourceVersion: "15007"
    uid: 4b50e14c-004c-4a4f-a2f1-31ee2d838c2a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68111fcbfe556
    namespace: kserve-ci-e2e-test
    resourceVersion: "15010"
    uid: 836c5114-c94d-4dfe-a4d0-bab25a334b09
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68111fd2b3ab2
    namespace: kserve-ci-e2e-test
    resourceVersion: "15011"
    uid: 4e23533c-ae4d-4a76-94f9-1d75dae80769
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68111fd351346
    namespace: kserve-ci-e2e-test
    resourceVersion: "15012"
    uid: 260e41ad-91a4-4c25-a53c-4293708385be
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c681120007e29f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15016"
    uid: 3813de2c-ec7f-414e-b97f-4deefbbe47cb
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68112006971d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "15017"
    uid: f0587ef7-1b87-4999-9fb2-eb8234309849
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:31:23Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:30:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:23Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c6811235ec3ef5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15530"
    uid: b9ea9e66-7552-4f01-be72-a259e5464df8
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c681281eab92db
    namespace: kserve-ci-e2e-test
    resourceVersion: "15839"
    uid: 9de21eb5-9b48-46bb-9de9-e299611ab151
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c681281eac299c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15840"
    uid: 843f0da7-8e18-45f2-b905-fed901278ef8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n
    namespace: kserve-ci-e2e-test
    resourceVersion: "14900"
    uid: 762283e4-6452-422c-ba38-a5c5f65f1503
  kind: Event
  lastTimestamp: "2026-07-28T16:31:53Z"
  message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp
    10.134.0.20:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:31:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:53Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n.18c68128903cb6ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "15952"
    uid: bdc581dc-3ab2-49c0-b336-b9397f118154
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14874"
    uid: ab6f86fe-fa4d-4252-bb12-b6d6e3d1dc7e
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: 'Created pod: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b-lpm5n'
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b.18c68110c86f8670
    namespace: kserve-ci-e2e-test
    resourceVersion: "14899"
    uid: e9ca3241-fded-4a32-962d-f7af5ae087f1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14871"
    uid: f94482a1-1774-4b2b-89a0-b28bc614cade
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-3b17e-predictor-6c84db749b
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor.18c68110c24b8609
    namespace: kserve-ci-e2e-test
    resourceVersion: "14875"
    uid: 884868bf-fa41-4ec2-a513-1a9f07e95673
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14873"
    uid: 46eb2890-256f-4126-b360-a7075e6e0680
  kind: Event
  lastTimestamp: "2026-07-28T16:30:41Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:30:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:41Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor.18c68114411c1d9e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15217"
    uid: be29baa6-0371-4b61-8bce-ca24cd41ba67
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14873"
    uid: 46eb2890-256f-4126-b360-a7075e6e0680
  kind: Event
  lastTimestamp: "2026-07-28T16:30:41Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:30:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:41Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor.18c681144189ab8e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15218"
    uid: 1347a973-9b18-42ed-8fe1-904beb2c2070
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:56Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15103"
    uid: 46eb2890-256f-4126-b360-a7075e6e0680
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:30:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor.18c6811b3f986ab9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15700"
    uid: 7361b011-e851-47f1-849d-71ac9b2717eb
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:56Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15103"
    uid: 46eb2890-256f-4126-b360-a7075e6e0680
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:30:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: isvc-sklearn-graph-raw-3b17e-predictor.18c6811b40226591
    namespace: kserve-ci-e2e-test
    resourceVersion: "15701"
    uid: 57caa241-145a-4018-8111-55b29027b727
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-3b17e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14860"
    uid: 1f747a2c-b917-4263-9315-ad86fe1f9929
  kind: Event
  lastTimestamp: "2026-07-28T16:31:28Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:31:28Z"
    name: isvc-sklearn-graph-raw-3b17e.18c68110c251d0c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15574"
    uid: ad8cf68f-475e-4170-b961-8e1fac2a8387
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:28Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-3b17e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15572"
    uid: 1f747a2c-b917-4263-9315-ad86fe1f9929
  kind: Event
  lastTimestamp: "2026-07-28T16:31:28Z"
  message: InferenceService [isvc-sklearn-graph-raw-3b17e] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:31:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:31:28Z"
    name: isvc-sklearn-graph-raw-3b17e.18c68122af025abc
    namespace: kserve-ci-e2e-test
    resourceVersion: "15573"
    uid: 8a0cb05b-33a1-49fb-ab77-7528385112df
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:31:51.678133Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15850"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c68128229989e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15856"
    uid: a1ce8525-a15d-450f-854a-6ae27e66e4fb
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-0983c-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c6812830355706
    namespace: kserve-ci-e2e-test
    resourceVersion: "15910"
    uid: 5ec72dea-138a-4999-9152-96ab85ae732d
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15865"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: Add eth0 [10.134.0.22/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681285fc21539
    namespace: kserve-ci-e2e-test
    resourceVersion: "15926"
    uid: 42b1789d-9b3b-4a2d-a657-cd4cf49940c6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681286156a09e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15928"
    uid: a793c02d-01ae-4f6e-892e-c8a84c9cb866
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c6812864e42cbd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15930"
    uid: 0f8d3607-4414-4475-8d18-6f3192f54280
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c68128655dc5ed
    namespace: kserve-ci-e2e-test
    resourceVersion: "15931"
    uid: 9d313795-498f-481d-a2a7-48b1c475aec3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:31:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:56Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681295c6b189b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15980"
    uid: f52b9faf-a6ae-4fb1-b327-97a238804954
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:31:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:56Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681295f6c4cf2
    namespace: kserve-ci-e2e-test
    resourceVersion: "15985"
    uid: 6e540473-3b64-440c-b6cc-92c516406f65
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681295fe404ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "15988"
    uid: b61e350b-0389-42b4-87c4-d6c844a85f27
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681295feee82d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15990"
    uid: 3de5f752-9513-4d89-aa3a-62688ac810f2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c681296276c972
    namespace: kserve-ci-e2e-test
    resourceVersion: "15993"
    uid: 53d55491-1e4f-4e69-889b-11ff5468cad1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c6812962e4eecc
    namespace: kserve-ci-e2e-test
    resourceVersion: "15996"
    uid: 5afd16cb-c0c2-485e-98f4-74f0ac7460a0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:33:43Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:43Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c6812998b93861
    namespace: kserve-ci-e2e-test
    resourceVersion: "16946"
    uid: 19948bdc-39ae-4ec0-a543-347b25b76f0f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:33:41Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c68141cf86ed57
    namespace: kserve-ci-e2e-test
    resourceVersion: "16883"
    uid: 0c0ce9d8-b60b-4194-86e4-49d0dc949e91
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:33:41Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c68141cf881619
    namespace: kserve-ci-e2e-test
    resourceVersion: "16885"
    uid: 29b73a65-b1d8-48f9-aa1e-95177af06373
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs
    namespace: kserve-ci-e2e-test
    resourceVersion: "15853"
    uid: d9638e0d-a4c7-47aa-8730-101a5ea12ffb
  kind: Event
  lastTimestamp: "2026-07-28T16:33:43Z"
  message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp
    10.134.0.22:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:33:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:43Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs.18c68142472f86c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16945"
    uid: 8959f749-c2d7-461c-b1ae-9b3c5bf73f6b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15823"
    uid: a1179783-f36e-4556-b665-79f28f527c3f
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: 'Created pod: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6-hmzbs'
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6.18c68128220f74f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15852"
    uid: 184b6ae1-9106-4e42-9bf0-314e072d7643
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15817"
    uid: 38300855-13da-4152-8f27-f09386008097
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-hpa-0983c-predictor-6dc5957ff6
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor.18c681281d060342
    namespace: kserve-ci-e2e-test
    resourceVersion: "15828"
    uid: 7220eced-d635-416e-bae5-b8fc774e7a04
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15818"
    uid: 373040d7-5f38-48ae-a332-d4ff153333f5
  kind: Event
  lastTimestamp: "2026-07-28T16:32:21Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:32:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:32:21Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor.18c6812b9b31505d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16197"
    uid: ba6af042-abe1-4aad-9ecb-c19e4ca686db
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15818"
    uid: 373040d7-5f38-48ae-a332-d4ff153333f5
  kind: Event
  lastTimestamp: "2026-07-28T16:32:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:32:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:32:21Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor.18c6812b9b9b9332
    namespace: kserve-ci-e2e-test
    resourceVersion: "16198"
    uid: 295f5c14-abf9-44a4-b49d-178ac67e0dc4
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16083"
    uid: 373040d7-5f38-48ae-a332-d4ff153333f5
  kind: Event
  lastTimestamp: "2026-07-28T16:33:21Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:32:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:21Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor.18c6813299c87d10
    namespace: kserve-ci-e2e-test
    resourceVersion: "16651"
    uid: f2b3080c-3e93-45b0-a40e-5e8a1a7ec7ad
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16083"
    uid: 373040d7-5f38-48ae-a332-d4ff153333f5
  kind: Event
  lastTimestamp: "2026-07-28T16:33:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:32:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:21Z"
    name: isvc-sklearn-graph-raw-hpa-0983c-predictor.18c681329a3096cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16652"
    uid: c85ea87e-a200-4f5a-89e4-6e1f83ba0f88
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-0983c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15809"
    uid: 32be7751-dcb8-46d8-91fb-4bedb97df5fb
  kind: Event
  lastTimestamp: "2026-07-28T16:33:14Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:14Z"
    name: isvc-sklearn-graph-raw-hpa-0983c.18c681281cf7eae1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16595"
    uid: e546317a-09ef-4551-98ac-107b7726730f
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:14Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-0983c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16592"
    uid: 32be7751-dcb8-46d8-91fb-4bedb97df5fb
  kind: Event
  lastTimestamp: "2026-07-28T16:33:14Z"
  message: InferenceService [isvc-sklearn-graph-raw-hpa-0983c] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:33:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:14Z"
    name: isvc-sklearn-graph-raw-hpa-0983c.18c6813b5138488e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16594"
    uid: 2e036f84-8d19-4183-b154-2e406524a154
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:35:27.373633Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17840"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815a5b0d9fde
    namespace: kserve-ci-e2e-test
    resourceVersion: "17846"
    uid: e8e88bf9-473e-4b68-8308-35d347fcadac
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17851"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Add eth0 [10.134.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815a748770cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17879"
    uid: df84c1f0-ab79-4eb5-85df-908dd229b6fb
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815a7678c1f1
    namespace: kserve-ci-e2e-test
    resourceVersion: "17881"
    uid: 98307e67-9aa5-4fe2-a013-2395bad9e0ca
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815a78c53d0c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17882"
    uid: 9cefd5e5-2fe6-4062-97d8-a550ccbd7490
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815a79287255
    namespace: kserve-ci-e2e-test
    resourceVersion: "17883"
    uid: c8fb1292-0943-4972-925a-d2236392dbdf
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:31Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:35:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:31Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815b64749f4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17920"
    uid: cd0ec373-1ff1-47fb-aa62-92a271ce14d8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:31Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:35:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:31Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815b6886b920
    namespace: kserve-ci-e2e-test
    resourceVersion: "17922"
    uid: f06483d9-386b-4aa4-97f1-83d5acbcc5f9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:31Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:35:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:31Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815b68ffbdef
    namespace: kserve-ci-e2e-test
    resourceVersion: "17925"
    uid: c7a30508-3051-43fc-bb3a-464cd83d34d5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:31Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:35:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:31Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815b6909feaf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17926"
    uid: cab1b9ec-1004-4782-92ee-741a3f9bd900
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:31Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:35:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:31Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815b6ba43ac0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17928"
    uid: c0c016c9-8ec1-49f1-9f65-828fa04f5f53
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:35:31Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:35:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:31Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815b6c180a3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17929"
    uid: 47a6a375-a943-4bca-8942-c45f73e03824
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 18
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:38:01Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:35:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:01Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6815ba06bcf77
    namespace: kserve-ci-e2e-test
    resourceVersion: "19018"
    uid: 00404a61-7016-4b33-9a03-bb2fe8a942ac
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:37:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:37:57Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:37:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:37:57Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6817d53026670
    namespace: kserve-ci-e2e-test
    resourceVersion: "18989"
    uid: b5fbf35d-08a4-4785-b1ab-03471a791065
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:37:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:37:57Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:37:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:37:57Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6817d5302da43
    namespace: kserve-ci-e2e-test
    resourceVersion: "18990"
    uid: 24b55271-f94d-4e7e-8e2b-f6c6188364ea
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:37:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c2b9b87e-bfbd-48dc-8208-e927dd60e1d3
  kind: Event
  lastTimestamp: "2026-07-28T16:38:04Z"
  message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp
    10.134.0.26:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:37:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:38:04Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n.18c6817ddae40ab3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19124"
    uid: 9178a411-c0c9-4ecb-a8a4-05e97061773b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58
    namespace: kserve-ci-e2e-test
    resourceVersion: "17818"
    uid: b27f6a40-453c-4250-bf23-a89fe888ef01
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: 'Created pod: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58-9n28n'
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58.18c6815a5a6c9102
    namespace: kserve-ci-e2e-test
    resourceVersion: "17843"
    uid: 2e01d251-5e19-4b4f-b379-256dd2c0a34e
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-scale-raw-6d6fe-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17811"
    uid: 74a13af2-5c66-42c0-96d6-0a7a2736a0c2
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Scaled up replica set isvc-sklearn-scale-raw-6d6fe-predictor-7f8c689f58
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor.18c6815a55b046c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17823"
    uid: d528f549-9fea-4fd1-a31f-424c404774d8
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:42Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6d6fe-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17817"
    uid: e5c03630-50b7-4a43-bd5a-9bec56238184
  kind: Event
  lastTimestamp: "2026-07-28T16:36:12Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:35:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:36:12Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor.18c6815dd426b258
    namespace: kserve-ci-e2e-test
    resourceVersion: "18226"
    uid: cfd7e4ec-f1d7-4720-9c6f-c48b2070543f
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:42Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6d6fe-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17817"
    uid: e5c03630-50b7-4a43-bd5a-9bec56238184
  kind: Event
  lastTimestamp: "2026-07-28T16:36:12Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:35:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:36:12Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor.18c6815dd48fc814
    namespace: kserve-ci-e2e-test
    resourceVersion: "18227"
    uid: d31548ad-c2a7-4dbd-b02f-eff86adb8698
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-07-28T16:36:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6d6fe-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18015"
    uid: e5c03630-50b7-4a43-bd5a-9bec56238184
  kind: Event
  lastTimestamp: "2026-07-28T16:37:57Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:36:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:37:57Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor.18c6816851cf9859
    namespace: kserve-ci-e2e-test
    resourceVersion: "18973"
    uid: b05f4b17-9514-4597-95a7-80e59ed42775
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-07-28T16:36:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6d6fe-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18015"
    uid: e5c03630-50b7-4a43-bd5a-9bec56238184
  kind: Event
  lastTimestamp: "2026-07-28T16:37:57Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:36:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:37:57Z"
    name: isvc-sklearn-scale-raw-6d6fe-predictor.18c68168523f53b7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18974"
    uid: 5ce4591b-2188-4ac4-8cb6-8b8c2e368a1a
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-6d6fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "17802"
    uid: f8ab4209-ed15-4f11-85ef-f63042198080
  kind: Event
  lastTimestamp: "2026-07-28T16:37:51Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:37:51Z"
    name: isvc-sklearn-scale-raw-6d6fe.18c6815a55b251cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "18933"
    uid: 6eba4719-8d72-4c2c-88ce-2af465a9d8a5
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-6d6fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "17826"
    uid: f8ab4209-ed15-4f11-85ef-f63042198080
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-6d6fe":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-6d6fe":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: isvc-sklearn-scale-raw-6d6fe.18c6815a59a64967
    namespace: kserve-ci-e2e-test
    resourceVersion: "17838"
    uid: 5a10a26b-6659-4a78-81c9-b92a26f400cb
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:37:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-6d6fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "18930"
    uid: f8ab4209-ed15-4f11-85ef-f63042198080
  kind: Event
  lastTimestamp: "2026-07-28T16:37:51Z"
  message: InferenceService [isvc-sklearn-scale-raw-6d6fe] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:37:51Z"
    name: isvc-sklearn-scale-raw-6d6fe.18c6817bdc41d2d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18932"
    uid: 480df292-417f-48cc-8b86-7800051a3f22
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:30:11.511973Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14930"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68110d03b4207
    namespace: kserve-ci-e2e-test
    resourceVersion: "14934"
    uid: f360a916-9623-468c-9085-8e0a972c142f
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14941"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Add eth0 [10.134.0.21/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68110ea51da46
    namespace: kserve-ci-e2e-test
    resourceVersion: "14956"
    uid: 6c09d362-4335-4658-bbf5-94882a3d372e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68110ec0070e0
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: c7b337df-a812-49d8-9996-422dc5dc4b40
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:12Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:30:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:12Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68110eed77331
    namespace: kserve-ci-e2e-test
    resourceVersion: "14961"
    uid: a7067509-ec95-439d-af8b-0f3b34ee92ed
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:12Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:30:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:12Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68110ef3dc326
    namespace: kserve-ci-e2e-test
    resourceVersion: "14962"
    uid: 12d51c77-14aa-44ae-8104-6446c205ff85
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:16Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-07-28T16:30:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:16Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68111f9bab227
    namespace: kserve-ci-e2e-test
    resourceVersion: "15006"
    uid: 544e5add-ddc4-4156-a997-28294714e750
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:35Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.113s (19.113s
    including waiting). Image size: 1307355896 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:30:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:35Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c681166cff966a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15172"
    uid: ed945b6c-b20c-4845-a092-49245c99e223
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:35Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:30:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:35Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68116702a1a20
    namespace: kserve-ci-e2e-test
    resourceVersion: "15173"
    uid: 871167f1-2fcb-4fb5-b7a8-7ac671f7b2a0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:35Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:30:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:35Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c6811670bc1365
    namespace: kserve-ci-e2e-test
    resourceVersion: "15174"
    uid: 31e7d54f-11f3-4f8b-8bb8-5a38b183db0b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:35Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:30:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:35Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c6811670c58ceb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15175"
    uid: 385df16b-e08a-4f00-b25f-9df791853aa3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:35Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:30:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:35Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c6811673277af5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15176"
    uid: 1b390e30-a772-4b22-acf1-2f2652657bb3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:30:35Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:30:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:30:35Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c681167388bacb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15177"
    uid: a8c683ec-0734-42c7-bd94-81fc8195b0a3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:30:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c68116a85d6237
    namespace: kserve-ci-e2e-test
    resourceVersion: "15925"
    uid: 1d5a6cd8-1181-46c2-8a3c-f31f71d44f49
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c681282ffc7cdd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15907"
    uid: ab8e3e60-713c-4f10-9246-4594dd7b058a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c681282ffce734
    namespace: kserve-ci-e2e-test
    resourceVersion: "15909"
    uid: d8aeca28-aabd-48be-ba6b-b96cf3f3be05
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14932"
    uid: 4f53a9b9-082f-4642-813d-50f90170d837
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp
    10.134.0.21:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b.18c681285a855310
    namespace: kserve-ci-e2e-test
    resourceVersion: "15924"
    uid: dd2d31b4-98db-4a92-8000-97ba9733a7d9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97
    namespace: kserve-ci-e2e-test
    resourceVersion: "14920"
    uid: 684690b9-171c-45e5-b6a5-d0739817dbe4
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: 'Created pod: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97-s7m4b'
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97.18c68110cfc6ea5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "14931"
    uid: 5b4624f0-a1d6-4b61-bf4e-a2dd5a52b3a1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14918"
    uid: a8f1d30d-d499-46bd-9e43-c9068d13a3dc
  kind: Event
  lastTimestamp: "2026-07-28T16:30:11Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-3b17e-predictor-86bddfdd97
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:30:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor.18c68110cc787c43
    namespace: kserve-ci-e2e-test
    resourceVersion: "14921"
    uid: 6103c37b-4630-4261-9c2a-061ebc20676d
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14919"
    uid: 8f47e982-bec6-46b2-bb15-aa01fcddca99
  kind: Event
  lastTimestamp: "2026-07-28T16:31:11Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:30:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor.18c681144b143a19
    namespace: kserve-ci-e2e-test
    resourceVersion: "15442"
    uid: a3154e01-bfc0-4aea-9f06-d84a36f52554
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14919"
    uid: 8f47e982-bec6-46b2-bb15-aa01fcddca99
  kind: Event
  lastTimestamp: "2026-07-28T16:31:11Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:30:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:11Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor.18c681144ba69df8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15443"
    uid: 26ec4d0c-9097-492c-9235-4b85952ba7c5
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15107"
    uid: 8f47e982-bec6-46b2-bb15-aa01fcddca99
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:31:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor.18c6812246c91714
    namespace: kserve-ci-e2e-test
    resourceVersion: "15707"
    uid: 7bd36bd3-e733-4395-8792-815e0d7057f4
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-3b17e-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15107"
    uid: 8f47e982-bec6-46b2-bb15-aa01fcddca99
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:31:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: isvc-xgboost-graph-raw-3b17e-predictor.18c68122473292e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15708"
    uid: 60e2d778-da04-4e10-b181-8c9e90edcd66
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:30:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-3b17e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14895"
    uid: babde358-51de-49c1-8202-bbd2683403c7
  kind: Event
  lastTimestamp: "2026-07-28T16:31:32Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:30:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:31:32Z"
    name: isvc-xgboost-graph-raw-3b17e.18c68110cc7eed0f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15615"
    uid: cbc2bf42-61c4-415a-964a-4814a10eee8d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-3b17e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15613"
    uid: babde358-51de-49c1-8202-bbd2683403c7
  kind: Event
  lastTimestamp: "2026-07-28T16:31:32Z"
  message: InferenceService [isvc-xgboost-graph-raw-3b17e] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:31:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:31:32Z"
    name: isvc-xgboost-graph-raw-3b17e.18c68123b8f58d5d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15614"
    uid: 46874015-d232-43ae-b5a6-f9ae7d9bef29
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:31:51.786847Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15874"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c68128291458c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15885"
    uid: 7682dd34-3797-4219-8fd1-6189076923bc
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-0983c-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c68128365951f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15916"
    uid: be8166b9-a38f-4119-8eb3-c5b11f8dbfba
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15888"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: Add eth0 [10.134.0.23/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c68128660ed87e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15932"
    uid: 948a3bcf-81b2-4ef5-b5b0-080bfd86c8e4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c6812867a589e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15934"
    uid: 20719d25-ca4e-415e-b53e-d18bb880c94e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681286a38aad2
    namespace: kserve-ci-e2e-test
    resourceVersion: "15935"
    uid: 7e9102dc-41a2-4fc9-b4a0-9be28e431bc1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:31:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:52Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681286a9c0fa2
    namespace: kserve-ci-e2e-test
    resourceVersion: "15936"
    uid: 4b1e6970-ab50-4fbd-ac00-db994f975af8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:56Z"
  message: Container image "kserve/xgbserver:latest" already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:31:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:56Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681295ca2e043
    namespace: kserve-ci-e2e-test
    resourceVersion: "15981"
    uid: 6c517aa0-1c59-4442-8143-171ee218d5c8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681295f7b3ffe
    namespace: kserve-ci-e2e-test
    resourceVersion: "15987"
    uid: bce83759-0b8c-4d86-8333-57c02a9f77c3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681295fe9bcbb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15989"
    uid: 9c63373c-2599-41af-b157-b4fcf7826259
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681295ff654c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15992"
    uid: b227a6e8-02f8-4b0d-9cc9-acc2aee7ec19
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c681296287ff89
    namespace: kserve-ci-e2e-test
    resourceVersion: "15994"
    uid: 9e1e7e3f-927e-43a2-b223-64a5602c0961
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:31:57Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:57Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c6812962fe77e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15998"
    uid: 7bc6bcde-bab5-4353-b85c-207c8faddd95
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:33:43Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:31:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:43Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c6812998dd48ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "16947"
    uid: cc0562c0-e63a-4dc0-8024-5e9f692eee83
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:33:42Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:33:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:42Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c68141d811acbf
    namespace: kserve-ci-e2e-test
    resourceVersion: "16921"
    uid: 48111e85-131f-4727-bb2b-38b66b5b1101
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:33:42Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:33:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:42Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c68141d8120e56
    namespace: kserve-ci-e2e-test
    resourceVersion: "16922"
    uid: f76e7723-24a1-4ac0-940f-3525221e8711
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 133fafda-789d-47b5-99a4-f7397419f336
  kind: Event
  lastTimestamp: "2026-07-28T16:33:43Z"
  message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp
    10.134.0.23:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:33:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:43Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk.18c68142472f0fbd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16944"
    uid: 9077fe21-0da5-4c58-ae13-fee94388ab83
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15864"
    uid: b284a158-1ae1-4fc9-b1dc-63dcf5b03218
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: 'Created pod: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5-gpqzk'
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5.18c68128283f508d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15877"
    uid: b48d0237-a729-47f7-85aa-462a1bd954cb
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15860"
    uid: a0d51fb3-0eaf-447a-af96-0891787ec3a3
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-hpa-0983c-predictor-76d6b6fdd5
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor.18c68128244de51c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15867"
    uid: 946b5ff6-72ce-4768-9d92-693180d60e58
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15863"
    uid: 47d814e9-d671-4152-b464-f18f34623c97
  kind: Event
  lastTimestamp: "2026-07-28T16:32:06Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:32:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:32:06Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor.18c6812ba2d1483d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16086"
    uid: fe8d8805-067b-4bc0-9bb6-2137803d7901
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15863"
    uid: 47d814e9-d671-4152-b464-f18f34623c97
  kind: Event
  lastTimestamp: "2026-07-28T16:32:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:32:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:32:06Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor.18c6812ba370809e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16088"
    uid: c5cbe482-358e-4f11-9113-6ac21e565e7e
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16087"
    uid: 47d814e9-d671-4152-b464-f18f34623c97
  kind: Event
  lastTimestamp: "2026-07-28T16:33:06Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:32:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:06Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor.18c6812f222f454e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16535"
    uid: 65812b19-314c-4b4d-ab13-720904e0b4ff
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16087"
    uid: 47d814e9-d671-4152-b464-f18f34623c97
  kind: Event
  lastTimestamp: "2026-07-28T16:33:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:32:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:06Z"
    name: isvc-xgboost-graph-raw-hpa-0983c-predictor.18c6812f229d6560
    namespace: kserve-ci-e2e-test
    resourceVersion: "16536"
    uid: ac4be7d8-f9e4-466c-b5d4-565f5eaf9c29
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-0983c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15843"
    uid: 678299dd-fce0-4640-807e-aa58e9ffb04d
  kind: Event
  lastTimestamp: "2026-07-28T16:32:54Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:32:54Z"
    name: isvc-xgboost-graph-raw-hpa-0983c.18c68128248bfcdc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16447"
    uid: ae8aa545-022f-4073-8c29-344d387758ee
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:32:54Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-0983c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16445"
    uid: 678299dd-fce0-4640-807e-aa58e9ffb04d
  kind: Event
  lastTimestamp: "2026-07-28T16:32:54Z"
  message: InferenceService [isvc-xgboost-graph-raw-hpa-0983c] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:32:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:32:54Z"
    name: isvc-xgboost-graph-raw-hpa-0983c.18c68136a8f70107
    namespace: kserve-ci-e2e-test
    resourceVersion: "16446"
    uid: ce5efa0d-ce10-4d77-92e3-3e5269d19a38
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:33:41.985392Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16887"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c68141d16ca4b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16893"
    uid: 1994e413-d3ee-4da9-98a3-a1eb6f1c19aa
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:42Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:42Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-9ed13-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:33:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:42Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c68141d8425f95
    namespace: kserve-ci-e2e-test
    resourceVersion: "16923"
    uid: 3da1bfa1-cded-4270-89f7-8a84230dbeb1
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:43Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16901"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:43Z"
  message: Add eth0 [10.134.0.24/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:33:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:33:43Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c681420ea8bd95
    namespace: kserve-ci-e2e-test
    resourceVersion: "16937"
    uid: bf719b76-8518-4261-8d81-3b49e272f825
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:43Z"
  message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
  metadata:
    creationTimestamp: "2026-07-28T16:33:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:43Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c6814210925543
    namespace: kserve-ci-e2e-test
    resourceVersion: "16939"
    uid: e9603d23-3157-4dc0-b666-96a932b5e644
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:44Z"
  message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    in 1.054s (1.054s including waiting). Image size: 14813193 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:44Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c681424f6b89e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16949"
    uid: 918c81a8-d85a-44a3-8e3e-449032d7f0d2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:44Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:44Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c6814252072022
    namespace: kserve-ci-e2e-test
    resourceVersion: "16950"
    uid: 36149f1c-6863-4b2f-9093-4d4bdd62642c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:44Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:44Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c68142526d660d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16951"
    uid: 3f3fee63-c908-43fe-accc-7430ad4e51b5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:44Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:44Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c681425276f698
    namespace: kserve-ci-e2e-test
    resourceVersion: "16952"
    uid: e10784dd-6579-4594-bf82-209a4dc9955a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:44Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:44Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c681425579bd01
    namespace: kserve-ci-e2e-test
    resourceVersion: "16954"
    uid: 5fc60a11-6470-4a1b-a50b-54eacab6a777
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:33:44Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:44Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c6814255e53c2f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16956"
    uid: db17a352-fd46-4e6d-bdee-78434d079cff
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c6815a6086d6aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "17869"
    uid: 53e581af-a583-4cf5-b42c-6686693104d9
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:35:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: f06b92fd-2110-4e92-bcae-c0e6bb9ff08e
  kind: Event
  lastTimestamp: "2026-07-28T16:35:27Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:35:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:35:27Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2.18c6815a608770d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17870"
    uid: 39ea52dc-f471-4767-ab6b-d58580272500
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-9ed13-predictor-577847d7b6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16868"
    uid: 8a3bf3b7-12e9-4607-8520-f5f70d8cbe15
  kind: Event
  lastTimestamp: "2026-07-28T16:33:41Z"
  message: 'Created pod: message-dumper-raw-9ed13-predictor-577847d7b6-4vgn2'
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: message-dumper-raw-9ed13-predictor-577847d7b6.18c68141d0d9a607
    namespace: kserve-ci-e2e-test
    resourceVersion: "16888"
    uid: 1e94e88a-9c51-4a5a-a90b-7eb2c7080903
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16862"
    uid: a9811bcd-1ce5-4a9f-9e43-da70fdd1a0a9
  kind: Event
  lastTimestamp: "2026-07-28T16:33:41Z"
  message: Scaled up replica set message-dumper-raw-9ed13-predictor-577847d7b6 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: message-dumper-raw-9ed13-predictor.18c68141cd903368
    namespace: kserve-ci-e2e-test
    resourceVersion: "16871"
    uid: cd4a78e4-5468-47f9-86fd-8a0346186669
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:56Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16867"
    uid: 9a89a9c7-3b22-405a-965d-e56c56a0c340
  kind: Event
  lastTimestamp: "2026-07-28T16:34:11Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:33:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:34:11Z"
    name: message-dumper-raw-9ed13-predictor.18c681454d792344
    namespace: kserve-ci-e2e-test
    resourceVersion: "17248"
    uid: 26ab1867-2c52-451d-9ca3-971ba34f5ffb
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:56Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-9ed13-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16867"
    uid: 9a89a9c7-3b22-405a-965d-e56c56a0c340
  kind: Event
  lastTimestamp: "2026-07-28T16:34:11Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:33:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:34:11Z"
    name: message-dumper-raw-9ed13-predictor.18c681454dc0485e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17249"
    uid: 701f90de-b168-4929-aaae-d5afcdfdcaee
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-9ed13
    namespace: kserve-ci-e2e-test
    resourceVersion: "16855"
    uid: 060639f9-b3ca-4c80-a2e3-a48d5aad19af
  kind: Event
  lastTimestamp: "2026-07-28T16:33:51Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:51Z"
    name: message-dumper-raw-9ed13.18c68141cda8300a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17028"
    uid: 3ce6c5c0-6689-4098-812a-af31eda39870
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-9ed13
    namespace: kserve-ci-e2e-test
    resourceVersion: "16878"
    uid: 060639f9-b3ca-4c80-a2e3-a48d5aad19af
  kind: Event
  lastTimestamp: "2026-07-28T16:33:41Z"
  message: 'Failed to update status for InferenceService "message-dumper-raw-9ed13":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-9ed13":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: message-dumper-raw-9ed13.18c68141d21a9973
    namespace: kserve-ci-e2e-test
    resourceVersion: "16900"
    uid: 35e1a8f4-3ce0-4708-bd1c-14baa12504a4
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-9ed13
    namespace: kserve-ci-e2e-test
    resourceVersion: "17025"
    uid: 060639f9-b3ca-4c80-a2e3-a48d5aad19af
  kind: Event
  lastTimestamp: "2026-07-28T16:33:51Z"
  message: InferenceService [message-dumper-raw-9ed13] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:33:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:51Z"
    name: message-dumper-raw-9ed13.18c6814407a2eac0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17026"
    uid: 2dcb3263-b20a-45f8-832b-baa45e44d441
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:31:41.378566Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15688"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-3b17e-776bc4b57f-v47hj
    to ip-10-0-132-37.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:31:41Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c68125bcb2ac45
    namespace: kserve-ci-e2e-test
    resourceVersion: "15694"
    uid: b73b58f4-5bd9-4ab1-9c06-bf34632c0edd
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:41Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15698"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: Add eth0 [10.133.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:31:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c68125d5fd2b8b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15711"
    uid: c351bdeb-4fd5-4a56-b5ed-ff9e2e06165f
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-3b17e}
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: Pulling image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2"
  metadata:
    creationTimestamp: "2026-07-28T16:31:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c68125d75a8f79
    namespace: kserve-ci-e2e-test
    resourceVersion: "15713"
    uid: 2f0ba99d-0f7c-4d9a-9e11-1919162139d3
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-3b17e}
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:31:44Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2"
    in 2.5s (2.5s including waiting). Image size: 218226814 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T16:31:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:44Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c681266c6942e1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15731"
    uid: f1d518d7-aa8a-40db-8e4f-323bc27bfb91
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-3b17e}
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:31:44Z"
  message: 'Created container: model-chainer-raw-3b17e'
  metadata:
    creationTimestamp: "2026-07-28T16:31:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:44Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c681266f092ad5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15733"
    uid: d19ec606-6906-4205-af0e-07bc113a0463
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-3b17e}
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:31:44Z"
  message: Started container model-chainer-raw-3b17e
  metadata:
    creationTimestamp: "2026-07-28T16:31:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:44Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c681266f6ab2e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15734"
    uid: f32daaba-3da5-4e41-a279-73e2aca16a55
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-3b17e}
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:31:51Z"
  message: Stopping container model-chainer-raw-3b17e
  metadata:
    creationTimestamp: "2026-07-28T16:31:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:31:51Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c6812813f7404f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15803"
    uid: 700af60b-8052-467b-88de-1cc64436df2f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-3b17e}
    kind: Pod
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: 8d22d3de-b8eb-4508-b870-f548779e1877
  kind: Event
  lastTimestamp: "2026-07-28T16:32:20Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T16:31:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:32:20Z"
    name: model-chainer-raw-3b17e-776bc4b57f-v47hj.18c6812914aa427f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16187"
    uid: b8bcd61a-d749-42af-a092-44adba5bb1e9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-3b17e-776bc4b57f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15685"
    uid: 9a993ef8-19bf-4d65-b555-001931501d26
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: 'Created pod: model-chainer-raw-3b17e-776bc4b57f-v47hj'
  metadata:
    creationTimestamp: "2026-07-28T16:31:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: model-chainer-raw-3b17e-776bc4b57f.18c68125bc314daa
    namespace: kserve-ci-e2e-test
    resourceVersion: "15690"
    uid: b177df96-04c2-4182-a440-b010a1b0c0ff
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-3b17e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15683"
    uid: b93d0e8a-4dfb-4be3-b918-f115b40cee55
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: Scaled up replica set model-chainer-raw-3b17e-776bc4b57f from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:31:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: model-chainer-raw-3b17e.18c68125bb0fdd9b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15686"
    uid: c8315aad-bf1c-4403-9fdd-36d366182108
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:31:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-3b17e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15703"
    uid: 551caeda-aca3-4d23-91ca-d6cc37da99a3
  kind: Event
  lastTimestamp: "2026-07-28T16:31:41Z"
  message: InferenceGraph [model-chainer-raw-3b17e] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:31:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:31:41Z"
    name: model-chainer-raw-3b17e.18c68125c2896bfe
    namespace: kserve-ci-e2e-test
    resourceVersion: "15705"
    uid: 6c3f1b58-628c-48b2-ac78-4975990b125a
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:33:31.715726Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16739"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    to ip-10-0-132-37.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:33:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:33:31Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6813f6d4e1689
    namespace: kserve-ci-e2e-test
    resourceVersion: "16743"
    uid: 87f8eef4-f3c4-4114-811d-21b8e69e488c
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:31Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16741"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:33:31Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-0983c-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T16:33:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:31Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6813f786e708c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16754"
    uid: 5cf843c5-c07d-45a6-a861-35b306383e97
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16747"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:33:32Z"
  message: Add eth0 [10.133.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:33:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:33:32Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6813faa2898f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16762"
    uid: 3867e8c9-362d-4d0a-9611-5a581a8d9775
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-0983c}
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16741"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:33:32Z"
  message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:33:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:32Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6813fab7b4c80
    namespace: kserve-ci-e2e-test
    resourceVersion: "16764"
    uid: db8e6d0f-a0f8-47c8-8946-4ce2aa3f125c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-0983c}
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16741"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:33:32Z"
  message: 'Created container: model-chainer-raw-hpa-0983c'
  metadata:
    creationTimestamp: "2026-07-28T16:33:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:32Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6813faec359ba
    namespace: kserve-ci-e2e-test
    resourceVersion: "16767"
    uid: cc855c8b-059e-4feb-a7bc-93a5c7ca44d5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-0983c}
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16741"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:33:32Z"
  message: Started container model-chainer-raw-hpa-0983c
  metadata:
    creationTimestamp: "2026-07-28T16:33:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:32Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6813faf22070c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16768"
    uid: 05181080-8da4-4af5-b4ab-cb34571fdd8e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-0983c}
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16741"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:33:41Z"
  message: Stopping container model-chainer-raw-hpa-0983c
  metadata:
    creationTimestamp: "2026-07-28T16:33:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:33:41Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c68141c3f4bdaf
    namespace: kserve-ci-e2e-test
    resourceVersion: "16849"
    uid: 459bb2dd-e97a-4026-8ba2-0b60d31d1c48
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-0983c}
    kind: Pod
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns
    namespace: kserve-ci-e2e-test
    resourceVersion: "16741"
    uid: 9ce1b21e-8d55-41f1-bb0e-2c37bd872ec7
  kind: Event
  lastTimestamp: "2026-07-28T16:34:09Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T16:33:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:34:09Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns.18c6814255d8b4bd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17229"
    uid: 6c2ed507-085f-4273-980c-aee00f612503
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-132-37.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-132-37.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-hpa-0983c-5dd8d85684
    namespace: kserve-ci-e2e-test
    resourceVersion: "16735"
    uid: 19897943-63c2-42da-870f-e09393e3e595
  kind: Event
  lastTimestamp: "2026-07-28T16:33:31Z"
  message: 'Created pod: model-chainer-raw-hpa-0983c-5dd8d85684-xmsns'
  metadata:
    creationTimestamp: "2026-07-28T16:33:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:31Z"
    name: model-chainer-raw-hpa-0983c-5dd8d85684.18c6813f6cd2973e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16740"
    uid: 832fa3a0-e851-41ab-a693-d45b71875937
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-hpa-0983c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16733"
    uid: 1ae8e555-2c01-4729-bed5-cce96bb2f4f1
  kind: Event
  lastTimestamp: "2026-07-28T16:33:31Z"
  message: Scaled up replica set model-chainer-raw-hpa-0983c-5dd8d85684 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T16:33:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:33:31Z"
    name: model-chainer-raw-hpa-0983c.18c6813f6b3b35ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "16736"
    uid: 2d342a3f-75f8-45f0-b96f-faba16325c8f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:33:31Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-0983c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16749"
    uid: 43d81c10-a9c9-4955-a83d-6a7152f2df93
  kind: Event
  lastTimestamp: "2026-07-28T16:33:31Z"
  message: InferenceGraph [model-chainer-raw-hpa-0983c] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:33:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:33:31Z"
    name: model-chainer-raw-hpa-0983c.18c6813f716708b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16751"
    uid: e9a6cd09-8be7-433b-b21e-517052b354f0
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:49:23.130883Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24783"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-3c435-predictor-864444488d-74727
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821cf2115d06
    namespace: kserve-ci-e2e-test
    resourceVersion: "24787"
    uid: 243bded7-661f-401e-9f87-cf33198fd846
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24793"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: Add eth0 [10.134.0.38/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821d0ba01a68
    namespace: kserve-ci-e2e-test
    resourceVersion: "24798"
    uid: 2fe110a9-da33-4dad-a0aa-644b29f8c066
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821d0d2cdd24
    namespace: kserve-ci-e2e-test
    resourceVersion: "24802"
    uid: af15d49e-c6cb-4ad0-a7bf-6c7f381c42ad
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821d0fb68954
    namespace: kserve-ci-e2e-test
    resourceVersion: "24803"
    uid: c488aa70-5dfd-45bc-9281-5d9e274f5c54
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821d10276f6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24804"
    uid: fb81a9aa-5675-4ede-ab38-7f6a4952bd3b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:28Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:49:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:28Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e1d1cff1f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24840"
    uid: c02aa05f-104a-4dfe-87d5-1a8c2e947abb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:28Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:49:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:28Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e2010655f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24842"
    uid: dd19c7db-5d1b-4c45-a682-4efc5b3001c9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:28Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:49:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:28Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e20857a46
    namespace: kserve-ci-e2e-test
    resourceVersion: "24843"
    uid: 6ac14e54-8ed0-4e65-be78-71c000230f55
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:28Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:49:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:28Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e208edeca
    namespace: kserve-ci-e2e-test
    resourceVersion: "24844"
    uid: 2a01a664-90c7-43ce-8cd3-7706f92ba985
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:28Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:49:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:28Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e2343c875
    namespace: kserve-ci-e2e-test
    resourceVersion: "24847"
    uid: 691b7274-4db7-4b8a-b56f-9fd48491e5a2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:49:28Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:49:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:49:28Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e23b84b7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24848"
    uid: ea7748e8-ced8-418d-a852-a00a8180dbeb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:50:46Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:49:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:46Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6821e94f2a581
    namespace: kserve-ci-e2e-test
    resourceVersion: "25713"
    uid: 64df3e71-dfe5-4e5b-9acb-6db92d0da1c4
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:50:43Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:50:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:43Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6822f996156b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "25688"
    uid: 3b45820a-d25b-4ee6-9378-517bc0fbf8b8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:50:43Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:50:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:43Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6822f9967986c
    namespace: kserve-ci-e2e-test
    resourceVersion: "25690"
    uid: 59a618ba-8c7e-4db1-a785-37654eb7e9fa
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-3c435-predictor-864444488d-74727
    namespace: kserve-ci-e2e-test
    resourceVersion: "24785"
    uid: a2f57fff-28a3-4087-9aec-b48326d7c4f0
  kind: Event
  lastTimestamp: "2026-07-28T16:50:46Z"
  message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp
    10.134.0.38:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:50:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:46Z"
    name: raw-sklearn-3c435-predictor-864444488d-74727.18c6823047102144
    namespace: kserve-ci-e2e-test
    resourceVersion: "25712"
    uid: c59af7f7-b555-453d-b38f-b7b5edbe4526
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-3c435-predictor-864444488d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24771"
    uid: 339a0c32-1715-4568-a3e1-32d5c65b6027
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: 'Created pod: raw-sklearn-3c435-predictor-864444488d-74727'
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor-864444488d.18c6821cf19ccfaa
    namespace: kserve-ci-e2e-test
    resourceVersion: "24784"
    uid: 079e9dc2-f47b-4167-a894-e5426eeb594d
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-3c435-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24769"
    uid: 2b744abf-5364-4527-bd3c-3f96c3a32757
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: Scaled up replica set raw-sklearn-3c435-predictor-864444488d from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435-predictor.18c6821cee2fb136
    namespace: kserve-ci-e2e-test
    resourceVersion: "24773"
    uid: 9bb0a91b-698a-4e50-9ef6-f15d9d185c9a
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:38Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-3c435-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24770"
    uid: e4e63a7a-f913-4650-9750-c686e85f2f47
  kind: Event
  lastTimestamp: "2026-07-28T16:49:38Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:49:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:49:38Z"
    name: raw-sklearn-3c435-predictor.18c682206cb9dcde
    namespace: kserve-ci-e2e-test
    resourceVersion: "24923"
    uid: 9c38d911-d074-4157-b2e5-d33108cefc3a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:38Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-3c435-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24770"
    uid: e4e63a7a-f913-4650-9750-c686e85f2f47
  kind: Event
  lastTimestamp: "2026-07-28T16:49:38Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:49:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:49:38Z"
    name: raw-sklearn-3c435-predictor.18c682206d32c9db
    namespace: kserve-ci-e2e-test
    resourceVersion: "24925"
    uid: 6a833a45-e29f-416c-b11c-8843a717f93f
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:53Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-3c435-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24924"
    uid: e4e63a7a-f913-4650-9750-c686e85f2f47
  kind: Event
  lastTimestamp: "2026-07-28T16:50:38Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:49:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:50:38Z"
    name: raw-sklearn-3c435-predictor.18c68223ec75f8aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "25502"
    uid: 134e8b96-4a36-4812-8611-e5a89f98c172
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:53Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-3c435-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24924"
    uid: e4e63a7a-f913-4650-9750-c686e85f2f47
  kind: Event
  lastTimestamp: "2026-07-28T16:50:38Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:49:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:50:38Z"
    name: raw-sklearn-3c435-predictor.18c68223ecf19bb1
    namespace: kserve-ci-e2e-test
    resourceVersion: "25503"
    uid: 4e274971-7585-40e6-a603-481f477bf512
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-3c435
    namespace: kserve-ci-e2e-test
    resourceVersion: "24761"
    uid: bbda035e-714c-4601-a204-c816ed6f7138
  kind: Event
  lastTimestamp: "2026-07-28T16:50:36Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:50:36Z"
    name: raw-sklearn-3c435.18c6821cee33d587
    namespace: kserve-ci-e2e-test
    resourceVersion: "25441"
    uid: 67dfdb83-d28d-4914-8046-30251eab3aed
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:49:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-3c435
    namespace: kserve-ci-e2e-test
    resourceVersion: "24761"
    uid: bbda035e-714c-4601-a204-c816ed6f7138
  kind: Event
  lastTimestamp: "2026-07-28T16:49:23Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-3c435": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-3c435":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:49:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:49:23Z"
    name: raw-sklearn-3c435.18c6821ceef2a401
    namespace: kserve-ci-e2e-test
    resourceVersion: "24778"
    uid: 035a7388-1393-454c-a687-57f7f660bcd1
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-3c435
    namespace: kserve-ci-e2e-test
    resourceVersion: "25439"
    uid: bbda035e-714c-4601-a204-c816ed6f7138
  kind: Event
  lastTimestamp: "2026-07-28T16:50:36Z"
  message: InferenceService [raw-sklearn-3c435] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:50:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:50:36Z"
    name: raw-sklearn-3c435.18c6822df993d760
    namespace: kserve-ci-e2e-test
    resourceVersion: "25440"
    uid: cb3b31ea-67b9-4016-ac19-6825c4cda319
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T16:50:48.510811Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25759"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    to ip-10-0-133-120.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T16:50:48Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68230d31b8a7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "25764"
    uid: 46eb0dd4-7c75-4250-9559-6538a2da29c0
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-8698cf6bb6-fz7hf
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:48Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25767"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:48Z"
  message: Add eth0 [10.134.0.39/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T16:50:48Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68230ec50d446
    namespace: kserve-ci-e2e-test
    resourceVersion: "25779"
    uid: e5f6fd43-12bf-4e10-8bcd-ad5c2fc0f9c4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:48Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:48Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68230edf1bf1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "25781"
    uid: dd374bc6-86fc-468b-bd45-e8e6b24d1813
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:49Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T16:50:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:49Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68230f0845e5e
    namespace: kserve-ci-e2e-test
    resourceVersion: "25782"
    uid: a49efea6-0aa6-4cbe-becb-70b0a41d0f99
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:49Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T16:50:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:49Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68230f0f6ca79
    namespace: kserve-ci-e2e-test
    resourceVersion: "25783"
    uid: 6413c9c5-7b50-4df1-92d1-45b256d2cced
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:53Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:53Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68231fab8bc32
    namespace: kserve-ci-e2e-test
    resourceVersion: "25823"
    uid: d19dc7df-73ba-4c99-9f32-eb53ad2987cc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:53Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T16:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:53Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68231fd90bc5e
    namespace: kserve-ci-e2e-test
    resourceVersion: "25826"
    uid: 352e79dc-5757-4641-961a-e639ad09bcab
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:53Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:53Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68231fe05db92
    namespace: kserve-ci-e2e-test
    resourceVersion: "25827"
    uid: 0ac38176-4518-48bf-9c37-0fa46fa6feb2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:53Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T16:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:53Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c68231fe0e616a
    namespace: kserve-ci-e2e-test
    resourceVersion: "25828"
    uid: 409766a3-c41b-457d-b327-9daf83f1a0e1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:53Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T16:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:53Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c6823200b0ed46
    namespace: kserve-ci-e2e-test
    resourceVersion: "25831"
    uid: fba6a42c-ad47-4625-834d-c136b9e9a3c0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:50:53Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:50:53Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c682320131a049
    namespace: kserve-ci-e2e-test
    resourceVersion: "25832"
    uid: 0fbf743e-471d-4ac0-bbc9-122b287606d2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:52:11Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T16:50:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:52:11Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c6823272810bcc
    namespace: kserve-ci-e2e-test
    resourceVersion: "26419"
    uid: 0eaccac9-f8f4-4197-97ef-6fc15214fe85
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:52:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:52:08Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T16:52:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:52:08Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c6824373b7fa68
    namespace: kserve-ci-e2e-test
    resourceVersion: "26398"
    uid: 06bb6046-c13f-45b7-a82f-03b53f7cc8ba
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:52:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:52:08Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T16:52:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:52:08Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c6824373b87f48
    namespace: kserve-ci-e2e-test
    resourceVersion: "26399"
    uid: 6b1a1f4b-6ac7-4975-8df1-5506823663e8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:52:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z
    namespace: kserve-ci-e2e-test
    resourceVersion: "25762"
    uid: 24e5e65a-0df2-4e20-9758-24f976bcbf07
  kind: Event
  lastTimestamp: "2026-07-28T16:52:11Z"
  message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp
    10.134.0.39:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T16:52:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T16:52:11Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z.18c6824424aa026b
    namespace: kserve-ci-e2e-test
    resourceVersion: "26418"
    uid: 652ce3d7-c98b-4014-a917-cced703083f1
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-133-120.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-133-120.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:48Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb
    namespace: kserve-ci-e2e-test
    resourceVersion: "25747"
    uid: eeb767d3-3640-483f-a44f-3cdc02ac0993
  kind: Event
  lastTimestamp: "2026-07-28T16:50:48Z"
  message: 'Created pod: raw-sklearn-runtime-f5110-predictor-8655474ffb-gkd4z'
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:50:48Z"
    name: raw-sklearn-runtime-f5110-predictor-8655474ffb.18c68230d28966c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "25761"
    uid: ec5f52a1-56c3-4495-b608-f1320ee38f92
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:48Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-runtime-f5110-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25745"
    uid: a6027a18-27db-4c19-a2fb-fd776d2abaa7
  kind: Event
  lastTimestamp: "2026-07-28T16:50:48Z"
  message: Scaled up replica set raw-sklearn-runtime-f5110-predictor-8655474ffb from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:50:48Z"
    name: raw-sklearn-runtime-f5110-predictor.18c68230cbd2ad29
    namespace: kserve-ci-e2e-test
    resourceVersion: "25750"
    uid: f9aab3e6-d654-4bd3-ba29-e38467e5196a
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:51:03Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-f5110-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25746"
    uid: 6b43fa69-e118-4b34-92f2-10be7140ae49
  kind: Event
  lastTimestamp: "2026-07-28T16:51:33Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:51:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:51:33Z"
    name: raw-sklearn-runtime-f5110-predictor.18c682344ac43385
    namespace: kserve-ci-e2e-test
    resourceVersion: "26122"
    uid: ee28a66d-87cb-44c3-9dc0-313291456772
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T16:51:03Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-f5110-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25746"
    uid: 6b43fa69-e118-4b34-92f2-10be7140ae49
  kind: Event
  lastTimestamp: "2026-07-28T16:51:33Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T16:51:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:51:33Z"
    name: raw-sklearn-runtime-f5110-predictor.18c682344b2eec36
    namespace: kserve-ci-e2e-test
    resourceVersion: "26123"
    uid: 14602e88-2407-4899-9675-7fb21da906ca
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:51:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-f5110-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25902"
    uid: 6b43fa69-e118-4b34-92f2-10be7140ae49
  kind: Event
  lastTimestamp: "2026-07-28T16:52:03Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:51:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:52:03Z"
    name: raw-sklearn-runtime-f5110-predictor.18c68237ca70f7ed
    namespace: kserve-ci-e2e-test
    resourceVersion: "26344"
    uid: 4d5105e9-791b-48d3-8ca9-4344b92e1b82
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T16:51:18Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-f5110-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25902"
    uid: 6b43fa69-e118-4b34-92f2-10be7140ae49
  kind: Event
  lastTimestamp: "2026-07-28T16:52:03Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T16:51:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T16:52:03Z"
    name: raw-sklearn-runtime-f5110-predictor.18c68237cb01304f
    namespace: kserve-ci-e2e-test
    resourceVersion: "26345"
    uid: b7511249-5df1-444b-8750-040f1bcc32db
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-f5110
    namespace: kserve-ci-e2e-test
    resourceVersion: "25737"
    uid: 79353f0d-2632-479a-b214-3e7201d96ae1
  kind: Event
  lastTimestamp: "2026-07-28T16:52:01Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:52:01Z"
    name: raw-sklearn-runtime-f5110.18c68230cbd179b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "26329"
    uid: fb2c8559-39fc-4322-82bc-e93db7eca3c4
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:50:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-f5110
    namespace: kserve-ci-e2e-test
    resourceVersion: "25737"
    uid: 79353f0d-2632-479a-b214-3e7201d96ae1
  kind: Event
  lastTimestamp: "2026-07-28T16:50:48Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-runtime-f5110":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-f5110":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T16:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:50:48Z"
    name: raw-sklearn-runtime-f5110.18c68230ccd48808
    namespace: kserve-ci-e2e-test
    resourceVersion: "25755"
    uid: b660ce06-dd45-491d-a0b0-24dfb3a73782
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T16:52:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-f5110
    namespace: kserve-ci-e2e-test
    resourceVersion: "26325"
    uid: 79353f0d-2632-479a-b214-3e7201d96ae1
  kind: Event
  lastTimestamp: "2026-07-28T16:52:01Z"
  message: InferenceService [raw-sklearn-runtime-f5110] is Ready
  metadata:
    creationTimestamp: "2026-07-28T16:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T16:52:01Z"
    name: raw-sklearn-runtime-f5110.18c68241d6dfe632
    namespace: kserve-ci-e2e-test
    resourceVersion: "26327"
    uid: 118d9ac9-4660-48aa-8009-dfb718567f7d
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
kind: EventList
metadata:
  resourceVersion: "26658"