---
apiVersion: v1
items:
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:46:03.243620Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20110"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-cb85cf55c-vr79g
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d18d0a48b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "20115"
    uid: 8b945f9a-5b92-4630-b33d-2172293d5aa4
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20129"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Add eth0 [10.134.0.23/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d1a7a007e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20138"
    uid: 66cf2945-3890-4713-9b4b-4cf2c25e6512
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d1a917f24b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20141"
    uid: e34cbece-7d41-4670-847a-ba35e403b2be
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d1ac46bd05
    namespace: kserve-ci-e2e-test
    resourceVersion: "20145"
    uid: ebd30bb3-7ee0-45a2-9ad3-f54dd4ac33b3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d1acbea545
    namespace: kserve-ci-e2e-test
    resourceVersion: "20147"
    uid: 2137c6c0-cc2d-4264-8f5b-3b99e7c5704c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d293c81617
    namespace: kserve-ci-e2e-test
    resourceVersion: "20184"
    uid: 10cd8e6a-6bc5-491f-b9f3-c47313fc338f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d2975e90b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "20190"
    uid: a99c2149-16ad-4067-b24f-6ddc9fdebd5c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d297fbf994
    namespace: kserve-ci-e2e-test
    resourceVersion: "20194"
    uid: 9936c791-c9ec-4088-aaa7-b1d190f2d75a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d2980610da
    namespace: kserve-ci-e2e-test
    resourceVersion: "20196"
    uid: ee24a288-9c27-4720-bf87-bf9cda187acf
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d29b17d53a
    namespace: kserve-ci-e2e-test
    resourceVersion: "20198"
    uid: 9b5d55a0-6ab1-40d4-a4f1-16451b1cdd76
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d29bbb41f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "20200"
    uid: 04bf87aa-232b-473b-a27e-8987605bc8cf
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:47:33Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:46:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:33Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650d2d01eab33
    namespace: kserve-ci-e2e-test
    resourceVersion: "20854"
    uid: de2ede51-50c3-48f0-b818-4e4e824c20f5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:47:33Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:47:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:33Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650e67f4f31f2
    namespace: kserve-ci-e2e-test
    resourceVersion: "20848"
    uid: 7e392505-ec7d-412d-91c2-a87baab6afa6
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:47:33Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:47:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:33Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650e67f50a95f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20850"
    uid: dbd3b7d9-ad43-4dc0-b680-5188dd0e85f1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-cb85cf55c-vr79g
    namespace: kserve-ci-e2e-test
    resourceVersion: "20113"
    uid: 50e181fa-64d8-4a2a-99fe-d772679735b8
  kind: Event
  lastTimestamp: "2026-07-28T01:47:34Z"
  message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp
    10.134.0.23:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:47:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:34Z"
    name: isvc-canary-create-predictor-cb85cf55c-vr79g.18c650e6d69a386f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20862"
    uid: 53f5ab7f-6b2c-47bf-8caf-d628e8fb6cb5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-cb85cf55c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20089"
    uid: 9de1c644-742b-483d-9822-95ce6b545d3f
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: 'Created pod: isvc-canary-create-predictor-cb85cf55c-vr79g'
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor-cb85cf55c.18c650d18c17f7ee
    namespace: kserve-ci-e2e-test
    resourceVersion: "20111"
    uid: 02e3c3c8-d05c-4df0-aabf-e571b34471eb
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20088"
    uid: 748575fa-9c9f-4b72-8030-3ae6a5d39d9f
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Scaled up replica set isvc-canary-create-predictor-cb85cf55c from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-predictor.18c650d181d36e27
    namespace: kserve-ci-e2e-test
    resourceVersion: "20090"
    uid: 88f5b948-6245-449f-986d-13e4e7d89cd5
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:46:03.263364Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20114"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d18e37864a
    namespace: kserve-ci-e2e-test
    resourceVersion: "20121"
    uid: c2d217c7-cfa7-43f1-b94d-0742a7fb767a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20130"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Add eth0 [10.134.0.24/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d1a8c69bd9
    namespace: kserve-ci-e2e-test
    resourceVersion: "20140"
    uid: 5ab9c8e0-4cfb-41fd-9e99-12dff187ebae
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d1aa244e61
    namespace: kserve-ci-e2e-test
    resourceVersion: "20143"
    uid: 44c0b366-54f9-4814-bfa4-1510e6067f07
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d1ac52c901
    namespace: kserve-ci-e2e-test
    resourceVersion: "20146"
    uid: 4827f945-a7bd-430d-aa25-baafe796360d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d1accbff79
    namespace: kserve-ci-e2e-test
    resourceVersion: "20148"
    uid: 93b4b779-08a3-424b-9624-5e35bc3d6e99
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d293df4a33
    namespace: kserve-ci-e2e-test
    resourceVersion: "20185"
    uid: b9e5bf2e-4c96-40c5-9b2c-e3faf0edc93b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d29755235f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20189"
    uid: 2627e311-867a-43cf-b205-469e549f76a3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d297dcf165
    namespace: kserve-ci-e2e-test
    resourceVersion: "20191"
    uid: df4e6de1-d244-44d4-a2bd-b417215fd5a0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d297e7c13c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20192"
    uid: c56f68f0-b111-4127-a856-45143380510e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d29b025e81
    namespace: kserve-ci-e2e-test
    resourceVersion: "20197"
    uid: c30e59fb-ae50-4ba1-b611-494d1dfe8261
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:46:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d29ba51a2e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20199"
    uid: 25875e85-06a3-47f5-81f0-be20f9447089
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:47:19Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:46:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:19Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650d30c3c0ed5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20712"
    uid: dfe7afc3-de68-41b2-8a90-86582232a6b5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:47:33Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:47:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:33Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650e67ebd6654
    namespace: kserve-ci-e2e-test
    resourceVersion: "20846"
    uid: f09d4f68-c627-49ff-a2b1-37d86c4c18a6
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:47:33Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:47:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:33Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650e67ec0c4c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20847"
    uid: f3ed9938-bb2c-4f91-900c-87729c811127
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 08344546-16ab-455f-b3e1-90ae76708908
  kind: Event
  lastTimestamp: "2026-07-28T01:47:35Z"
  message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp
    10.134.0.24:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:47:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:35Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm.18c650e712737861
    namespace: kserve-ci-e2e-test
    resourceVersion: "20868"
    uid: b7d4b0f3-94bb-444c-8586-ac2aba34cd99
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-v2-predictor-6d8d4768d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "20098"
    uid: 2f4392b2-10ea-47ff-94aa-aaa3e46c9efc
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: 'Created pod: isvc-canary-create-v2-predictor-6d8d4768d4-d5nbm'
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor-6d8d4768d4.18c650d18da951bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20119"
    uid: 45cba4c4-2ef3-4a16-9e57-3d8dd2f294c1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20097"
    uid: ebf4b8c4-276c-43e8-9691-7ac7cc72dee4
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: Scaled up replica set isvc-canary-create-v2-predictor-6d8d4768d4 from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create-v2-predictor.18c650d183d60296
    namespace: kserve-ci-e2e-test
    resourceVersion: "20100"
    uid: 44c005fd-1f43-4cb5-a1ae-6def31a611cd
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 15
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20080"
    uid: f6045150-8be1-4cf8-b3fc-68c3fa227ba8
  kind: Event
  lastTimestamp: "2026-07-28T01:47:29Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:47:29Z"
    name: isvc-canary-create.18c650d18398fe55
    namespace: kserve-ci-e2e-test
    resourceVersion: "20795"
    uid: cd5051dd-8aa7-408a-b1fd-01c618261ff0
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20080"
    uid: f6045150-8be1-4cf8-b3fc-68c3fa227ba8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: 'Failed to update status for InferenceService "isvc-canary-create": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create.18c650d184e3cd87
    namespace: kserve-ci-e2e-test
    resourceVersion: "20102"
    uid: 45afde53-a99b-43a8-886a-1c1b07959069
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:46:03Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20080"
    uid: f6045150-8be1-4cf8-b3fc-68c3fa227ba8
  kind: Event
  lastTimestamp: "2026-07-28T01:46:03Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:46:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:46:03Z"
    name: isvc-canary-create.18c650d184e42889
    namespace: kserve-ci-e2e-test
    resourceVersion: "20105"
    uid: c6ba79a8-936d-4e40-96dc-9be5089452e2
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20744"
    uid: f6045150-8be1-4cf8-b3fc-68c3fa227ba8
  kind: Event
  lastTimestamp: "2026-07-28T01:47:23Z"
  message: InferenceService [isvc-canary-create] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:47:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:47:23Z"
    name: isvc-canary-create.18c650e43eb9f448
    namespace: kserve-ci-e2e-test
    resourceVersion: "20746"
    uid: e5cd730e-345b-4294-826c-961ebe2ced91
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:47:38.367248Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20918"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-d9759ffd5-85vv5
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e7b2d9ad4d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20923"
    uid: 5540b20f-f30c-4fad-b6de-a4306ab48a1e
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20932"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Add eth0 [10.134.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e7ccc4e334
    namespace: kserve-ci-e2e-test
    resourceVersion: "20949"
    uid: 56b21afe-28c5-4c78-a3d2-09413e86ba54
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e7ce35776e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20952"
    uid: 343664c4-e514-4b37-8543-6addf4cfa541
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e7d12f943e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20955"
    uid: 6d8e7f1a-2577-4355-887e-b32c240df624
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e7d1aab7c9
    namespace: kserve-ci-e2e-test
    resourceVersion: "20956"
    uid: 321ee51a-ffb1-42ea-848f-51c14855d726
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e8c8154ee8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21004"
    uid: 41f64c2e-2250-4c39-8d23-896c367ab474
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e8cbc25106
    namespace: kserve-ci-e2e-test
    resourceVersion: "21008"
    uid: fb585d51-6f93-4f2c-bebd-b36e250c1e9b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e8cc3adf11
    namespace: kserve-ci-e2e-test
    resourceVersion: "21010"
    uid: 220092e8-74bf-4996-93bf-9714562edee0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e8cc44e251
    namespace: kserve-ci-e2e-test
    resourceVersion: "21011"
    uid: 486e22ba-eac8-4be5-bc4e-1e22f77b5d98
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e8cf3125f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "21016"
    uid: 4988a0f7-93cd-4994-8ce7-aac141b5163b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e8cfb19976
    namespace: kserve-ci-e2e-test
    resourceVersion: "21018"
    uid: 680f306b-d459-474b-b399-76e7b35151bc
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:49:11Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:47:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:11Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650e94068f3c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "21694"
    uid: 497e5e4d-1553-4ea1-8b02-e7d42da45f97
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650fcac768f7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21650"
    uid: d881ebb8-ca79-453b-b4b4-8e5721f1d72b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650fcac76f85d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21651"
    uid: c62c762d-f1f1-4781-b945-332b47f66cbf
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: a80ae38c-a41e-43de-ad68-07b8a635956c
  kind: Event
  lastTimestamp: "2026-07-28T01:49:11Z"
  message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp
    10.134.0.25:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:49:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:11Z"
    name: isvc-canary-promote-predictor-d9759ffd5-85vv5.18c650fd4697d676
    namespace: kserve-ci-e2e-test
    resourceVersion: "21693"
    uid: 0c564d02-cf50-4338-8927-27f854dbd052
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-d9759ffd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20900"
    uid: ad44b401-5db4-4003-8a19-180eafa5ae86
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: 'Created pod: isvc-canary-promote-predictor-d9759ffd5-85vv5'
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor-d9759ffd5.18c650e7b2613730
    namespace: kserve-ci-e2e-test
    resourceVersion: "20919"
    uid: 7059b73a-0bc4-49ec-829b-f52ae2b232c9
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20899"
    uid: c36162bc-f0d3-4f15-9bc4-2e012292a394
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Scaled up replica set isvc-canary-promote-predictor-d9759ffd5 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-predictor.18c650e7ad64bb06
    namespace: kserve-ci-e2e-test
    resourceVersion: "20901"
    uid: 7b04b30f-4bdb-4b35-895b-cb3d64df15ff
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:49:08.554685Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21655"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-86766c4958-br2tz
    to ip-10-0-135-95.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c650fcb270ba32
    namespace: kserve-ci-e2e-test
    resourceVersion: "21661"
    uid: e46dab3d-9220-4535-a27a-986e6d1da463
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:08Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21667"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: Add eth0 [10.133.0.24/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c650fccb995d02
    namespace: kserve-ci-e2e-test
    resourceVersion: "21673"
    uid: 3cdf803c-34b1-4c90-a44f-63712f201621
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:09Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:49:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:09Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c650fccd0dd71d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21675"
    uid: fd8fb2c0-2ef9-4105-abe7-1e23beb7aed6
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:09Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:49:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:09Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c650fccff4ea73
    namespace: kserve-ci-e2e-test
    resourceVersion: "21676"
    uid: af707254-d70b-4c30-b33c-c0eb7042e666
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:09Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:49:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:09Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c650fcd06c5fe1
    namespace: kserve-ci-e2e-test
    resourceVersion: "21677"
    uid: 13f76dd8-b5ff-4e17-a048-9ae1c7d70be6
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:12Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
  metadata:
    creationTimestamp: "2026-07-28T01:49:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:12Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c650fdb6236db7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21709"
    uid: 0b8c7ec1-e386-4b45-a056-de8a3083d9f5
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:27Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    in 14.223s (14.223s including waiting). Image size: 1555730357 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:49:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:27Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c6510105f13011
    namespace: kserve-ci-e2e-test
    resourceVersion: "21815"
    uid: 9594bc80-6078-411a-a5db-8e3b60ff0cbe
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:27Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:49:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:27Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c6510109272925
    namespace: kserve-ci-e2e-test
    resourceVersion: "21816"
    uid: 6d0f49a5-af40-405a-92bd-9ff344747a30
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:27Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:49:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:27Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c65101099b46ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "21817"
    uid: e152c18a-4675-4fe2-ad92-87707e52033a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:27Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-07-28T01:49:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:27Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c6510109a8fd62
    namespace: kserve-ci-e2e-test
    resourceVersion: "21818"
    uid: f1274a8b-e28e-415c-90ff-af7fda0ab4b6
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:29Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.28s (2.28s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:49:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:29Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c651019198ef7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21835"
    uid: cb082670-1590-4964-99a6-c2f3b937737c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:29Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:49:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:29Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c65101944a28da
    namespace: kserve-ci-e2e-test
    resourceVersion: "21836"
    uid: 52617192-7923-4bc9-bbe9-6f7c84c05356
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:49:29Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:49:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:49:29Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c6510194be68a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "21837"
    uid: ae12a31a-5240-4e21-8fc6-27b90ed8cf03
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:50:50Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:49:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:50Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c65101ec135d9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22425"
    uid: a4d2a144-24d9-4e89-ab1b-eecf8a7d8a12
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:50:48Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:48Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c65114027a9265
    namespace: kserve-ci-e2e-test
    resourceVersion: "22407"
    uid: 262a0488-6922-48b8-a14e-17d0a2ea8a7e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:50:48Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:48Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c65114027b2da1
    namespace: kserve-ci-e2e-test
    resourceVersion: "22410"
    uid: f8d64dda-a166-48be-9ee9-bb24aef09e20
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz
    namespace: kserve-ci-e2e-test
    resourceVersion: "21659"
    uid: e2582450-c9e1-4dbf-b3b3-87ac9dd9cdb0
  kind: Event
  lastTimestamp: "2026-07-28T01:50:52Z"
  message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp
    10.133.0.24:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:50:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:52Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-br2tz.18c65114c85fe758
    namespace: kserve-ci-e2e-test
    resourceVersion: "22437"
    uid: c5de77a7-7783-4e1c-9d01-a75c2798215f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:47:38.379604Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20922"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e7b3962e26
    namespace: kserve-ci-e2e-test
    resourceVersion: "20928"
    uid: 83debdd3-7308-46d5-9473-e2ea8f4b5f4d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20939"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Add eth0 [10.134.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e7ce2583a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "20951"
    uid: b045554a-bcd1-49f7-8e25-8fc486cc1208
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e7d02dd5dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20954"
    uid: db77b75b-fcd4-41bb-b7db-db3bb79d06fc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e7d47934d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "20958"
    uid: f62d4460-ff81-4f30-8f2e-19bf5f7b14d6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e7d4f30ccd
    namespace: kserve-ci-e2e-test
    resourceVersion: "20959"
    uid: 04cc64bb-d00f-43c5-a79b-18599a5929b6
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e8c82c6918
    namespace: kserve-ci-e2e-test
    resourceVersion: "21005"
    uid: 330f90c5-ccc0-40c6-b995-90cc0d26354a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e8cbcfcbd6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21009"
    uid: 6b418ab4-fda3-48a0-bb6c-ac1a3ab174c1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e8cc47a19e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21013"
    uid: 25974dc1-fe0f-462f-a5b4-a22914422f6c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e8cc521a61
    namespace: kserve-ci-e2e-test
    resourceVersion: "21015"
    uid: 9ab8eca9-96b8-4276-b585-d29386df44f0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e8cf3c20f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21017"
    uid: 305f4b78-c31d-4ea3-a805-8948006cf818
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:47:43Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:47:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:47:43Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e8cfc05038
    namespace: kserve-ci-e2e-test
    resourceVersion: "21019"
    uid: b655b129-057b-4b2c-aabf-f422e71c6ad9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:50:50Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:47:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:50Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c650e9049d5dd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22421"
    uid: 00c80bfd-f6fe-4263-900e-0c43e4a5c4c8
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:50:48Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:48Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c6511402a266a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22408"
    uid: 2e5d5f32-67e8-4070-acc4-cdbfc0190586
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:50:48Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:50:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:48Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c6511402a30b96
    namespace: kserve-ci-e2e-test
    resourceVersion: "22409"
    uid: 0aee0f66-6f67-49ca-af55-538cc3829e67
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: 0eb29efa-bb9d-4001-9feb-120e7e6c7616
  kind: Event
  lastTimestamp: "2026-07-28T01:50:50Z"
  message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp
    10.134.0.26:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:50:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:50Z"
    name: isvc-canary-promote-v2-predictor-86766c4958-dhwdx.18c6511453390375
    namespace: kserve-ci-e2e-test
    resourceVersion: "22420"
    uid: 0d58e0de-4383-41f0-97f3-5710456085d0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-86766c4958
    namespace: kserve-ci-e2e-test
    resourceVersion: "20911"
    uid: 8ab4fc70-4e5c-4fd1-84d1-f363d30f5fe3
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-86766c4958-dhwdx'
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor-86766c4958.18c650e7b307d582
    namespace: kserve-ci-e2e-test
    resourceVersion: "20924"
    uid: 0094097c-76ae-4d7c-84e6-a5217570cf0c
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:08Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-86766c4958
    namespace: kserve-ci-e2e-test
    resourceVersion: "21640"
    uid: 8ab4fc70-4e5c-4fd1-84d1-f363d30f5fe3
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-86766c4958-br2tz'
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote-v2-predictor-86766c4958.18c650fcb1576161
    namespace: kserve-ci-e2e-test
    resourceVersion: "21657"
    uid: 7059e7eb-6b94-4881-a274-c14e55a7aceb
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20909"
    uid: 3c656eee-5d84-49e0-b7a1-2325e9e74e47
  kind: Event
  lastTimestamp: "2026-07-28T01:47:38Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-86766c4958 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:47:38Z"
    name: isvc-canary-promote-v2-predictor.18c650e7afe2f830
    namespace: kserve-ci-e2e-test
    resourceVersion: "20913"
    uid: 15a7e0f9-2e05-4c7f-b5a0-b954200323e6
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:08Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21635"
    uid: 3c656eee-5d84-49e0-b7a1-2325e9e74e47
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-86766c4958 from
    1 to 2
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote-v2-predictor.18c650fcab062778
    namespace: kserve-ci-e2e-test
    resourceVersion: "21645"
    uid: df1cf53e-07c6-4c5e-9dfb-b2ebe93c9090
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 20
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "20892"
    uid: cea97902-5608-4bba-a2fd-1519042cbb3f
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote.18c650e7afa3df8d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21669"
    uid: 12d3d4a3-0d60-4e53-9d88-149124580f68
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "20892"
    uid: cea97902-5608-4bba-a2fd-1519042cbb3f
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote.18c650e7b011849e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21656"
    uid: 1a9e27e5-2738-40f2-965c-eee5d04d78ce
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:47:38Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "20892"
    uid: cea97902-5608-4bba-a2fd-1519042cbb3f
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:47:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote.18c650e7b011da97
    namespace: kserve-ci-e2e-test
    resourceVersion: "21660"
    uid: 51756b60-4f69-4503-b2c3-5133745284d5
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "21582"
    uid: cea97902-5608-4bba-a2fd-1519042cbb3f
  kind: Event
  lastTimestamp: "2026-07-28T01:50:40Z"
  message: InferenceService [isvc-canary-promote] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:49:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:50:40Z"
    name: isvc-canary-promote.18c650faf960d7bc
    namespace: kserve-ci-e2e-test
    resourceVersion: "22336"
    uid: a60505bc-553d-4e2e-bb36-0e943397f585
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:49:08Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "21665"
    uid: cea97902-5608-4bba-a2fd-1519042cbb3f
  kind: Event
  lastTimestamp: "2026-07-28T01:49:08Z"
  message: 'InferenceService [isvc-canary-promote] is no longer Ready because of:
    PredictorReady, Ready, Stopped'
  metadata:
    creationTimestamp: "2026-07-28T01:49:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:49:08Z"
    name: isvc-canary-promote.18c650fcb4dae269
    namespace: kserve-ci-e2e-test
    resourceVersion: "21668"
    uid: b4911c10-49ab-4055-84b9-17d6285d0eb9
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:50:53.831432Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22481"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    to ip-10-0-135-95.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:50:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c65115356c6718
    namespace: kserve-ci-e2e-test
    resourceVersion: "22486"
    uid: 4b7690fa-7602-42e9-9b2b-451c8ba92c4a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22499"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: Add eth0 [10.133.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c651154f1aab67
    namespace: kserve-ci-e2e-test
    resourceVersion: "22512"
    uid: f941e882-df08-4249-9c7b-6cbc7858bdd1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c65115507fef46
    namespace: kserve-ci-e2e-test
    resourceVersion: "22516"
    uid: bb92daaf-a3d7-469b-99b1-d218b17385f3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6511553ca88b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "22518"
    uid: 8c75de51-23a2-4958-bd26-124c319eda9f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6511554423455
    namespace: kserve-ci-e2e-test
    resourceVersion: "22519"
    uid: 5d2555c8-d541-44f4-aff0-4d2f2d4b0a0e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c65116426ab8c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "22562"
    uid: b295e1bf-7d40-4093-a1ad-17005610b531
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c65116458b3e6b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22564"
    uid: 488ecc45-b5c9-4077-a1d2-b2cb70eb91cd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6511645fd5d86
    namespace: kserve-ci-e2e-test
    resourceVersion: "22565"
    uid: 9462244c-5d41-4771-8afe-eef2b4a1e7d4
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c651164606a439
    namespace: kserve-ci-e2e-test
    resourceVersion: "22566"
    uid: 8ac8c43c-4330-4f2b-8ef2-8fa65c68036c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6511648f7a3b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "22569"
    uid: 1e43a777-3a36-48df-8a5a-0b4794d7a2fd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c65116496a7d6e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22570"
    uid: 5079ce00-5dce-4589-b3ff-c724737d9be2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:51:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:53:46Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:51:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:46Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c65116ba3a8aea
    namespace: kserve-ci-e2e-test
    resourceVersion: "23855"
    uid: 3653a970-3bba-413a-ad7b-15354e65df7a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:53:43Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:53:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:43Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6513cd38464b8
    namespace: kserve-ci-e2e-test
    resourceVersion: "23825"
    uid: 31b6df90-81b3-4bc0-9753-61616e96dacc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:53:43Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:53:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:43Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6513cd385327f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23826"
    uid: 1b8cb0a8-ec9c-4449-a410-01f189535fb8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 68056153-71a5-4c56-b80d-1eca5e42fdb4
  kind: Event
  lastTimestamp: "2026-07-28T01:53:46Z"
  message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp
    10.133.0.25:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:53:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:46Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7.18c6513d60d705a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "23854"
    uid: e280dc5a-846b-48e8-86af-c73a317a80bc
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:52:23.937234Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23216"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:52:23Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512a3025e5f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23220"
    uid: 1c4bb582-2a77-4afe-8f66-e20cb20567fe
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:24Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23226"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:24Z"
  message: Add eth0 [10.134.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:52:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:52:24Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512a496f0dc9
    namespace: kserve-ci-e2e-test
    resourceVersion: "23236"
    uid: b4ec955c-0dbd-4d7e-8889-4d035e6e4e8b
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:24Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:52:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:24Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512a4ae0b607
    namespace: kserve-ci-e2e-test
    resourceVersion: "23238"
    uid: 6589f602-83f3-483e-be1b-7fd4225f51bc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:24Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:52:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:24Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512a4dec1807
    namespace: kserve-ci-e2e-test
    resourceVersion: "23239"
    uid: b3743b27-a881-4894-ad5a-a494d348aab4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:24Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:24Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:52:24Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:24Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512a4e6599cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "23240"
    uid: 3e6bd137-2c4a-4cc4-b636-7e5563ccf4ef
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:29Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:52:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:29Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b5f0fff5c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23279"
    uid: 7b15212c-f078-456a-b20b-a47f3bb842d8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:29Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:52:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:29Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b629e0602
    namespace: kserve-ci-e2e-test
    resourceVersion: "23284"
    uid: 5e44902a-77e6-4595-9f11-97bb7dcc5dfa
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:29Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:52:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:29Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b63205af0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23286"
    uid: 205296cf-8ccb-407e-9f16-190b2e0a4ddb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:29Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:52:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:29Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b632a3d90
    namespace: kserve-ci-e2e-test
    resourceVersion: "23287"
    uid: 5e843700-c185-4748-9893-d5d1595b5c6c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:29Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:52:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:29Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b65c3e918
    namespace: kserve-ci-e2e-test
    resourceVersion: "23288"
    uid: 6125be18-c93a-4e2f-9f47-aacd3ce16bbc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:52:29Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:52:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:29Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b663b14ac
    namespace: kserve-ci-e2e-test
    resourceVersion: "23289"
    uid: 6cdc04ee-4b50-4de0-8d43-3f52fe84e38f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:53:36Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:52:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:36Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6512b9b19b57d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23750"
    uid: 79defa90-3790-4712-99f2-2347b7eeb112
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:44Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:53:47Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-rollback-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:53:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:47Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6513ce797e91e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23864"
    uid: 55692bcf-0cbd-4103-98db-131e38655815
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:53:45Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:53:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:45Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6513d212892ee
    namespace: kserve-ci-e2e-test
    resourceVersion: "23843"
    uid: e2a1ec02-27d2-4399-96b0-8578d3cd5ebe
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:53:45Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:53:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:45Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6513d212904b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23844"
    uid: 099b04b1-c463-4423-a302-6df6c5bc250d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt
    namespace: kserve-ci-e2e-test
    resourceVersion: "23218"
    uid: deb86c1d-7f2c-456e-806c-dbe14b66b3a3
  kind: Event
  lastTimestamp: "2026-07-28T01:53:46Z"
  message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp
    10.134.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:53:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:46Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt.18c6513d4d2b0fd1
    namespace: kserve-ci-e2e-test
    resourceVersion: "23849"
    uid: b00e1380-1bee-4489-aece-c7592281ac87
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-5bbb8c8cfc
    namespace: kserve-ci-e2e-test
    resourceVersion: "22465"
    uid: 80e13143-a6ee-42e1-8546-2042648017a6
  kind: Event
  lastTimestamp: "2026-07-28T01:50:53Z"
  message: 'Created pod: isvc-canary-rollback-predictor-5bbb8c8cfc-msst7'
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:50:53Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc.18c65115350a601a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22482"
    uid: bf5b5a05-be00-42e9-88b2-59d79806bf54
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:23Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-5bbb8c8cfc
    namespace: kserve-ci-e2e-test
    resourceVersion: "23199"
    uid: 80e13143-a6ee-42e1-8546-2042648017a6
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: 'Created pod: isvc-canary-rollback-predictor-5bbb8c8cfc-v5rlt'
  metadata:
    creationTimestamp: "2026-07-28T01:52:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback-predictor-5bbb8c8cfc.18c6512a2fba0d6b
    namespace: kserve-ci-e2e-test
    resourceVersion: "23217"
    uid: 1850d9e3-8ccc-47e9-a35f-d9e4694cfe80
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22464"
    uid: c6161d2f-1ba8-4390-8b4f-c03aee034048
  kind: Event
  lastTimestamp: "2026-07-28T01:50:53Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-5bbb8c8cfc from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:50:53Z"
    name: isvc-canary-rollback-predictor.18c65115307eec41
    namespace: kserve-ci-e2e-test
    resourceVersion: "22466"
    uid: 0f3b6107-1a30-4c69-af35-1eab1a91ae81
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:23Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23194"
    uid: c6161d2f-1ba8-4390-8b4f-c03aee034048
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-5bbb8c8cfc from 1
    to 2
  metadata:
    creationTimestamp: "2026-07-28T01:52:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback-predictor.18c6512a2a8412d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23203"
    uid: c1cd6a26-853d-4530-acfa-2fc89366862c
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:50:53.842749Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22484"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:50:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651153619143d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22492"
    uid: fd19eb10-eb87-4926-84c4-8e85eeade714
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22500"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: Add eth0 [10.134.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651154f75ce8a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22513"
    uid: 24c21199-a087-494a-9317-53502e606326
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c6511550dc7d35
    namespace: kserve-ci-e2e-test
    resourceVersion: "22517"
    uid: 4676f641-396a-4fd9-b0f6-cf356586520c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c65115547bf594
    namespace: kserve-ci-e2e-test
    resourceVersion: "22520"
    uid: c6094c8f-a5bd-413b-abe1-ee3f1bc40bcd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:54Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:50:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:54Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c6511554f5f427
    namespace: kserve-ci-e2e-test
    resourceVersion: "22521"
    uid: 47ff635c-7aac-4c4e-a185-44fc1a4847be
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651165858a867
    namespace: kserve-ci-e2e-test
    resourceVersion: "22575"
    uid: 86e6f138-e6cb-4836-a487-5579aa807683
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651165b2d9020
    namespace: kserve-ci-e2e-test
    resourceVersion: "22577"
    uid: 3d5c94c0-cb4a-45f1-9def-86bc0508f713
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651165bc20b7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22578"
    uid: f9d5f98a-273a-477b-88d0-d2cb28dab3fa
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651165bcbfd48
    namespace: kserve-ci-e2e-test
    resourceVersion: "22579"
    uid: 25ae84d8-42f0-4629-a11c-9d199ed36e11
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651165e8c7cd4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22581"
    uid: 08170445-0e22-4da0-9501-ae5ff6ef36cc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:50:58Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:50:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:50:58Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c651165f0bd82b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22582"
    uid: 5e2f95a4-7daf-4dce-8a86-ca3c22f09599
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:52:25Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:50:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:25Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c65116945b42cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "23252"
    uid: e084c7ed-8338-4f5d-b96f-f6bafdda2d45
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:52:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c6512a2b808a63
    namespace: kserve-ci-e2e-test
    resourceVersion: "23210"
    uid: 002faef6-9873-48df-830b-3a1614fa4105
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:52:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c6512a2b821600
    namespace: kserve-ci-e2e-test
    resourceVersion: "23211"
    uid: 8a3b8de4-fe7c-4de9-bc80-6678c343b05e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv
    namespace: kserve-ci-e2e-test
    resourceVersion: "22488"
    uid: 683d2a54-8975-4d2b-9c7b-140bd8e7608c
  kind: Event
  lastTimestamp: "2026-07-28T01:52:25Z"
  message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp
    10.134.0.27:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:52:25Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv.18c6512a9a8970ba
    namespace: kserve-ci-e2e-test
    resourceVersion: "23251"
    uid: b6591719-7a4f-49c6-90f2-1a382f7a8bfc
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-v2-predictor-84f48b7656
    namespace: kserve-ci-e2e-test
    resourceVersion: "22474"
    uid: 3a0c4600-f173-4605-b0ba-4f9bf25e3491
  kind: Event
  lastTimestamp: "2026-07-28T01:50:53Z"
  message: 'Created pod: isvc-canary-rollback-v2-predictor-84f48b7656-sm5cv'
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:50:53Z"
    name: isvc-canary-rollback-v2-predictor-84f48b7656.18c6511535a57894
    namespace: kserve-ci-e2e-test
    resourceVersion: "22487"
    uid: 56b46a9d-eaee-4b82-85e4-fc04aeefc296
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22473"
    uid: af8f1660-32d8-4425-8da1-6e24d3eb7fa2
  kind: Event
  lastTimestamp: "2026-07-28T01:50:53Z"
  message: Scaled up replica set isvc-canary-rollback-v2-predictor-84f48b7656 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:50:53Z"
    name: isvc-canary-rollback-v2-predictor.18c651153286ff21
    namespace: kserve-ci-e2e-test
    resourceVersion: "22476"
    uid: ee40e91f-e8da-4350-9ca0-f79c4d506101
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 20
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "22454"
    uid: a0f6c7f6-59fa-4a53-b6fa-3d7b2f672feb
  kind: Event
  lastTimestamp: "2026-07-28T01:52:24Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:52:24Z"
    name: isvc-canary-rollback.18c65115323e4f63
    namespace: kserve-ci-e2e-test
    resourceVersion: "23231"
    uid: d46e227c-942a-4328-a2fb-de71e73e05da
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "22454"
    uid: a0f6c7f6-59fa-4a53-b6fa-3d7b2f672feb
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: 'Failed to update status for InferenceService "isvc-canary-rollback": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback.18c6511532cb7e9e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23228"
    uid: 95ad1d75-fce2-4340-8e75-cd64925b25dc
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:50:53Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "22454"
    uid: a0f6c7f6-59fa-4a53-b6fa-3d7b2f672feb
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:50:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback.18c6511532cbc490
    namespace: kserve-ci-e2e-test
    resourceVersion: "23229"
    uid: fd30ea13-c714-48ef-b418-6eeb2a343cad
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:16Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "23141"
    uid: a0f6c7f6-59fa-4a53-b6fa-3d7b2f672feb
  kind: Event
  lastTimestamp: "2026-07-28T01:53:43Z"
  message: InferenceService [isvc-canary-rollback] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:52:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:53:43Z"
    name: isvc-canary-rollback.18c651287325757f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23806"
    uid: 7140c429-5211-4c57-b3c4-c3cad3353dfb
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:52:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "23214"
    uid: a0f6c7f6-59fa-4a53-b6fa-3d7b2f672feb
  kind: Event
  lastTimestamp: "2026-07-28T01:52:23Z"
  message: 'InferenceService [isvc-canary-rollback] is no longer Ready because of:
    PredictorReady, Ready, Stopped'
  metadata:
    creationTimestamp: "2026-07-28T01:52:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:52:23Z"
    name: isvc-canary-rollback.18c6512a2f2d8e31
    namespace: kserve-ci-e2e-test
    resourceVersion: "23215"
    uid: 1bb37e27-0a86-4eef-b094-5d68f0f1f080
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:53:49.155682Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23906"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    to ip-10-0-135-95.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513e079026ed
    namespace: kserve-ci-e2e-test
    resourceVersion: "23912"
    uid: fcded0c8-f1ae-418c-806f-a1c6f981f468
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23917"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: Add eth0 [10.133.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513e21466281
    namespace: kserve-ci-e2e-test
    resourceVersion: "23936"
    uid: d1863584-45c9-49c7-83f9-d14e075570cb
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513e229c166c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23938"
    uid: 4a0165ab-ef2a-421a-ad12-f96a53ae4d89
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513e25d4c1e9
    namespace: kserve-ci-e2e-test
    resourceVersion: "23939"
    uid: b10d3942-e57a-4a21-96dc-9bff7b945df9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513e264b595c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23940"
    uid: d722324a-1bdf-4503-9b64-e6af7d61e759
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513f2a5e1226
    namespace: kserve-ci-e2e-test
    resourceVersion: "23987"
    uid: b36c8e71-7f06-40f6-9422-424fe1af50c3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513f2d6e4386
    namespace: kserve-ci-e2e-test
    resourceVersion: "23989"
    uid: f0c7fe34-01bd-4b3f-b372-776a6d2066d3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513f2de6d577
    namespace: kserve-ci-e2e-test
    resourceVersion: "23991"
    uid: f32cd22c-85f3-499c-883b-371ffffd34c0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513f2df1788e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23992"
    uid: bfcaf9f3-f9e6-4cb3-a013-b122ca27c32e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513f30d86f57
    namespace: kserve-ci-e2e-test
    resourceVersion: "23995"
    uid: 2d6a045a-6a5f-4d9c-955e-c8b8f4e712db
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513f315c440d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23996"
    uid: 8abfe753-988d-4b8b-a0d5-9e96d005bf60
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:55:22Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:22Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c6513fa23938a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "24672"
    uid: a7587794-1d08-49a7-b836-d378956aeb9d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:55:19Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:55:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:19Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c65152ff7df501
    namespace: kserve-ci-e2e-test
    resourceVersion: "24636"
    uid: 485c037b-8b03-484d-81ca-146a4fe403b1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:55:19Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:55:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:19Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c65152ff7eb25f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24638"
    uid: da0aae30-5de2-4d67-97f7-3bd0412ca310
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23909"
    uid: 17e0ffc7-5bca-4bc5-abb4-4ee926401337
  kind: Event
  lastTimestamp: "2026-07-28T01:55:22Z"
  message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp
    10.133.0.26:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:55:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:22Z"
    name: isvc-canary-stop-predictor-86dc9fdb65-vjtj4.18c65153a86f2fd3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24671"
    uid: 2f8d738f-8829-4290-9e3f-1c63669e6950
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-86dc9fdb65
    namespace: kserve-ci-e2e-test
    resourceVersion: "23890"
    uid: c0f95503-a7c3-40d1-ac61-cc489a2de61f
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: 'Created pod: isvc-canary-stop-predictor-86dc9fdb65-vjtj4'
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor-86dc9fdb65.18c6513e07158a72
    namespace: kserve-ci-e2e-test
    resourceVersion: "23908"
    uid: b77e9f25-7a62-4c7d-ac73-d6afac6130d9
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23888"
    uid: 0eb5ac9e-918b-46e5-bab5-d81b07bd7336
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: Scaled up replica set isvc-canary-stop-predictor-86dc9fdb65 from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-predictor.18c6513e03b4ef16
    namespace: kserve-ci-e2e-test
    resourceVersion: "23891"
    uid: bad6a14f-393e-4f31-975d-037565a68156
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:53:49.204769Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23918"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513e0a7d2d44
    namespace: kserve-ci-e2e-test
    resourceVersion: "23922"
    uid: 1981679f-c041-4cad-a340-58ac05de5acf
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513e140879d8
    namespace: kserve-ci-e2e-test
    resourceVersion: "23932"
    uid: 5f3c651d-2d6b-42ec-bba2-a8941599328c
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:50Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23928"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:50Z"
  message: Add eth0 [10.134.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:53:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:53:50Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513e4790716e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23951"
    uid: 0de56a13-abbb-4f29-afe3-8de9113d1f93
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:50Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:53:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:50Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513e48f1cb11
    namespace: kserve-ci-e2e-test
    resourceVersion: "23954"
    uid: ad320ed8-3054-47ca-ae4e-0c4bfc9ff373
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:50Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:53:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:50Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513e4b752059
    namespace: kserve-ci-e2e-test
    resourceVersion: "23955"
    uid: 11541b61-6da0-4ed0-9df2-21a76c097e28
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:50Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:53:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:50Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513e4bf840bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "23956"
    uid: 70b13a1f-9d31-48fb-a246-218f22cbc3ed
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f3be7838f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24001"
    uid: dc26f6de-0afd-4dbd-9432-9fb647f41c15
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f3f761094
    namespace: kserve-ci-e2e-test
    resourceVersion: "24003"
    uid: f3c56aba-f738-4ea7-ab3e-d78f6fe00489
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f3ff165b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "24004"
    uid: 9aa3f8d7-1bbc-4827-8d81-ebfe78f533bb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f3ffbbb6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24005"
    uid: 5ec6bdc1-6c46-48a9-8df0-eaee066c2218
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f42d88aa1
    namespace: kserve-ci-e2e-test
    resourceVersion: "24007"
    uid: 44e04657-76e1-42bf-943f-2a3c13dfe647
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:53:54Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:53:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:53:54Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f4352b550
    namespace: kserve-ci-e2e-test
    resourceVersion: "24008"
    uid: 153d8cd7-53f0-4bb4-a6b7-5cd50711884e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:55:01Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:53:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:01Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c6513f77d554b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "24481"
    uid: 214cf89a-a703-4046-94a1-425ec35c3819
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:55:19Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:55:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:19Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c65153019970a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24645"
    uid: 832b562c-6ace-4bf5-aa66-212a3f97d223
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:55:19Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:55:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:19Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c65153019a278c
    namespace: kserve-ci-e2e-test
    resourceVersion: "24646"
    uid: e9c384dd-86c4-4486-ba65-ea8e3b40652b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j
    namespace: kserve-ci-e2e-test
    resourceVersion: "23920"
    uid: 19af2665-2e6e-4110-9146-fa8ac990d0af
  kind: Event
  lastTimestamp: "2026-07-28T01:55:21Z"
  message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp
    10.134.0.29:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:55:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:21Z"
    name: isvc-canary-stop-v2-predictor-77f778d864-xkv9j.18c651537e08f50b
    namespace: kserve-ci-e2e-test
    resourceVersion: "24666"
    uid: a5be2894-1a9b-4e3b-871f-bebfc4af06a6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-v2-predictor-77f778d864
    namespace: kserve-ci-e2e-test
    resourceVersion: "23900"
    uid: 6bc9e7f5-3b2b-4629-8beb-6cea1ef26b88
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: 'Created pod: isvc-canary-stop-v2-predictor-77f778d864-xkv9j'
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-v2-predictor-77f778d864.18c6513e0a183ee5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23919"
    uid: 3214bb4f-50b5-4c70-9a3d-13d8c8b9b693
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23897"
    uid: 6e59ffda-c5ff-4ac3-bbe7-d249e6a4a46b
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: Scaled up replica set isvc-canary-stop-v2-predictor-77f778d864 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop-v2-predictor.18c6513e05d07f7a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23903"
    uid: c48f6e7a-f3a8-4c83-8ab8-55d8b7bf2815
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "23879"
    uid: 474c486e-1504-4582-8433-e2199646720c
  kind: Event
  lastTimestamp: "2026-07-28T01:55:19Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:55:19Z"
    name: isvc-canary-stop.18c6513e05a71c7a
    namespace: kserve-ci-e2e-test
    resourceVersion: "24653"
    uid: 7a75a297-8de1-4b6f-93f7-37ea518b5de2
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "23879"
    uid: 474c486e-1504-4582-8433-e2199646720c
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop.18c6513e067da206
    namespace: kserve-ci-e2e-test
    resourceVersion: "23905"
    uid: 1779d8d7-dc94-49fe-a896-0d854c5840a9
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:53:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "23879"
    uid: 474c486e-1504-4582-8433-e2199646720c
  kind: Event
  lastTimestamp: "2026-07-28T01:53:49Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:53:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:53:49Z"
    name: isvc-canary-stop.18c6513e067deef0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23907"
    uid: 3d73753d-8d02-49ff-8227-2924bb2bcc1a
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "24570"
    uid: 474c486e-1504-4582-8433-e2199646720c
  kind: Event
  lastTimestamp: "2026-07-28T01:55:12Z"
  message: InferenceService [isvc-canary-stop] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:55:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:55:12Z"
    name: isvc-canary-stop.18c651515b0fed9f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24571"
    uid: 12b3ac18-7e08-44cb-b128-f6166bffaf97
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:19Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "24642"
    uid: 474c486e-1504-4582-8433-e2199646720c
  kind: Event
  lastTimestamp: "2026-07-28T01:55:19Z"
  message: 'InferenceService [isvc-canary-stop] is no longer Ready because of: PredictorReady,
    Ready'
  metadata:
    creationTimestamp: "2026-07-28T01:55:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:55:19Z"
    name: isvc-canary-stop.18c6515301479c4d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24644"
    uid: 91f18d77-37e3-44f0-8996-259a338f35c3
  reason: InferenceServiceNotReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:57:26.010783Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh
    namespace: kserve-ci-e2e-test
    resourceVersion: "26041"
    uid: 84e577f0-d28a-41e1-8595-8156a550a69a
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-40119c-predictor-58899449b-b7mwh
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:57:26Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:57:26Z"
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh.18c6517085225b53
    namespace: kserve-ci-e2e-test
    resourceVersion: "26046"
    uid: dbff4e29-976d-4320-a7d1-55cf3f5985d3
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh
    namespace: kserve-ci-e2e-test
    resourceVersion: "26051"
    uid: 84e577f0-d28a-41e1-8595-8156a550a69a
  kind: Event
  lastTimestamp: "2026-07-28T01:57:26Z"
  message: Add eth0 [10.134.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:57:26Z"
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh.18c651709e86a235
    namespace: kserve-ci-e2e-test
    resourceVersion: "26060"
    uid: 587a1b65-ce82-423e-86fa-c77e75940d39
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh
    namespace: kserve-ci-e2e-test
    resourceVersion: "26042"
    uid: 84e577f0-d28a-41e1-8595-8156a550a69a
  kind: Event
  lastTimestamp: "2026-07-28T01:57:29Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:29Z"
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh.18c65170a00164b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "26094"
    uid: b621089f-fba3-41fd-9221-47ba3fac38f5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh
    namespace: kserve-ci-e2e-test
    resourceVersion: "26042"
    uid: 84e577f0-d28a-41e1-8595-8156a550a69a
  kind: Event
  lastTimestamp: "2026-07-28T01:57:29Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:29Z"
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh.18c65170a31e877f
    namespace: kserve-ci-e2e-test
    resourceVersion: "26097"
    uid: 91b29b0d-2370-4734-8cd4-7df7d2236793
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh
    namespace: kserve-ci-e2e-test
    resourceVersion: "26042"
    uid: 84e577f0-d28a-41e1-8595-8156a550a69a
  kind: Event
  lastTimestamp: "2026-07-28T01:57:29Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:29Z"
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh.18c65170a39335cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "26098"
    uid: 5c9807dd-6bea-49b9-83a1-f91705c181f3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh
    namespace: kserve-ci-e2e-test
    resourceVersion: "26042"
    uid: 84e577f0-d28a-41e1-8595-8156a550a69a
  kind: Event
  lastTimestamp: "2026-07-28T01:57:31Z"
  message: Stopping container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:57:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:31Z"
    name: isvc-init-fail-40119c-predictor-58899449b-b7mwh.18c65171b2f18b47
    namespace: kserve-ci-e2e-test
    resourceVersion: "26135"
    uid: 05837c5f-f6a9-4019-b138-067e2f1377d8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:26Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-init-fail-40119c-predictor-58899449b
    namespace: kserve-ci-e2e-test
    resourceVersion: "26034"
    uid: 65570784-7b51-4a91-9a35-1d039ebe591b
  kind: Event
  lastTimestamp: "2026-07-28T01:57:26Z"
  message: 'Created pod: isvc-init-fail-40119c-predictor-58899449b-b7mwh'
  metadata:
    creationTimestamp: "2026-07-28T01:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:26Z"
    name: isvc-init-fail-40119c-predictor-58899449b.18c6517084ba5658
    namespace: kserve-ci-e2e-test
    resourceVersion: "26044"
    uid: 44228f0c-d46b-4c09-8d30-f744cd433cac
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-init-fail-40119c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26031"
    uid: cd32e4d6-49dc-4484-8b71-16cd144c3666
  kind: Event
  lastTimestamp: "2026-07-28T01:57:25Z"
  message: Scaled up replica set isvc-init-fail-40119c-predictor-58899449b from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T01:57:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:25Z"
    name: isvc-init-fail-40119c-predictor.18c6517080f6c21e
    namespace: kserve-ci-e2e-test
    resourceVersion: "26035"
    uid: 2ab98805-3b24-499b-84c0-255607c1b7b0
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-40119c
    namespace: kserve-ci-e2e-test
    resourceVersion: "26024"
    uid: c432b5bc-6ed1-42e3-8c21-897239aef81d
  kind: Event
  lastTimestamp: "2026-07-28T01:57:29Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:57:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:29Z"
    name: isvc-init-fail-40119c.18c651708111a917
    namespace: kserve-ci-e2e-test
    resourceVersion: "26101"
    uid: 0ee0bcc8-758a-4251-8c28-7ee80728e8ea
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-40119c
    namespace: kserve-ci-e2e-test
    resourceVersion: "26024"
    uid: c432b5bc-6ed1-42e3-8c21-897239aef81d
  kind: Event
  lastTimestamp: "2026-07-28T01:57:25Z"
  message: 'Failed to update status for InferenceService "isvc-init-fail-40119c":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-40119c":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:57:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:25Z"
    name: isvc-init-fail-40119c.18c65170819b2d82
    namespace: kserve-ci-e2e-test
    resourceVersion: "26038"
    uid: a56858eb-af45-43f9-b22c-138d6d916594
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-40119c
    namespace: kserve-ci-e2e-test
    resourceVersion: "26024"
    uid: c432b5bc-6ed1-42e3-8c21-897239aef81d
  kind: Event
  lastTimestamp: "2026-07-28T01:57:25Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-init-fail-40119c": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:57:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:25Z"
    name: isvc-init-fail-40119c.18c65170819b87c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "26040"
    uid: bf4c0b28-3712-471a-994e-5c2379f2c13a
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:41:42.553265Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18030"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:41:42Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:41:42Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65094daaf1e20
    namespace: kserve-ci-e2e-test
    resourceVersion: "18036"
    uid: 901028bd-5afa-4a7a-b1ec-8b83590154cc
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:42Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18040"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:42Z"
  message: Add eth0 [10.134.0.21/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:41:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:41:42Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65094f464044a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18046"
    uid: 3cdf41aa-760e-4d49-9f27-3dd713b1bf89
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:43Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:41:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:43Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65094f5dc7ca2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18048"
    uid: 52148b92-bf70-4a0e-8433-8cb3e4575d00
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:43Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:41:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:43Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65094f8a5c34e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18049"
    uid: 9bc5aa51-3168-4043-8b82-3b2d60186266
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:43Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:41:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:43Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65094f915c77c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18050"
    uid: 7e3dcdc7-f8b7-48d5-b77f-b29a588c434c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650960e9b65fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "18092"
    uid: a24074a3-0673-4d46-994d-7c9652cd514b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650961187a14f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18095"
    uid: 3badade7-8560-4c41-8bb9-5e0cc932c044
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650961211e537
    namespace: kserve-ci-e2e-test
    resourceVersion: "18096"
    uid: c0de3852-db32-404d-94b6-29d2a17eb0b3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65096121c5297
    namespace: kserve-ci-e2e-test
    resourceVersion: "18097"
    uid: 055220f3-8aa1-431d-a5cc-8157c9641e43
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c6509615109526
    namespace: kserve-ci-e2e-test
    resourceVersion: "18099"
    uid: f9cc8585-6e98-4ce8-a2c9-b5a53bd7cb35
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c6509615880917
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: 9440a073-685e-4975-b0c4-fc6142a36dfa
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: Container image "quay.io/opendatahub/kserve-agent@sha256:bf42462edec7da298c1b007872394204ebe10a2c0c0d143725247988fb0a7324"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c6509615920b2c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18104"
    uid: 3781ad0b-bbb8-4a9a-8e84-73b21bb6767f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c65096186c7029
    namespace: kserve-ci-e2e-test
    resourceVersion: "18105"
    uid: 65b62923-159f-4872-b12f-34b730a5c25f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:41:47Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:47Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c6509618ee11d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18106"
    uid: f39ece06-dbac-46e9-89af-a8158335f029
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:43:34Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:41:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:34Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650964aa32794
    namespace: kserve-ci-e2e-test
    resourceVersion: "19005"
    uid: 3f4ae230-ac49-4e8c-83c8-389a8315ceab
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:43:24Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T01:41:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:24Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650964aadf744
    namespace: kserve-ci-e2e-test
    resourceVersion: "18930"
    uid: 6bb3733f-7d56-45d3-9326-4654760b7155
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650ab032810fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18806"
    uid: b91d7e93-ec8f-4554-a592-83d4dd53e98c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650ab03286ee4
    namespace: kserve-ci-e2e-test
    resourceVersion: "18809"
    uid: 243fef32-e3b2-4a6a-8660-fa60a10a68ca
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650ab0328aae7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18812"
    uid: 8f1467bd-0181-439f-8ad7-57c30a11e144
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18033"
    uid: 186f4dca-65e7-49c8-8476-21ebd5bca49d
  kind: Event
  lastTimestamp: "2026-07-28T01:43:34Z"
  message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp
    10.134.0.21:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:43:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:34Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b.18c650ab7ad5c870
    namespace: kserve-ci-e2e-test
    resourceVersion: "19004"
    uid: 5f8bbfe1-ac62-4f37-878f-e507797e70cd
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:42Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-18035-predictor-7cb99b957d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18017"
    uid: d730608b-6a33-46f1-8f07-adce3fbc4884
  kind: Event
  lastTimestamp: "2026-07-28T01:41:42Z"
  message: 'Created pod: isvc-logger-raw-18035-predictor-7cb99b957d-f6z4b'
  metadata:
    creationTimestamp: "2026-07-28T01:41:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:42Z"
    name: isvc-logger-raw-18035-predictor-7cb99b957d.18c65094d9d75587
    namespace: kserve-ci-e2e-test
    resourceVersion: "18031"
    uid: e7368e8b-5146-4071-9ee9-a7ffe731dc17
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:42Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18015"
    uid: 780e0c2f-6456-473f-ba33-942f62452257
  kind: Event
  lastTimestamp: "2026-07-28T01:41:42Z"
  message: Scaled up replica set isvc-logger-raw-18035-predictor-7cb99b957d from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T01:41:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:42Z"
    name: isvc-logger-raw-18035-predictor.18c65094d566d397
    namespace: kserve-ci-e2e-test
    resourceVersion: "18019"
    uid: 7df52d0a-367b-471b-89b3-a03d2eb03c48
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18016"
    uid: ab4d1f97-acc2-406b-9489-7a814755d284
  kind: Event
  lastTimestamp: "2026-07-28T01:42:27Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:41:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:42:27Z"
    name: isvc-logger-raw-18035-predictor.18c65098540ac0b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18402"
    uid: 22c2360a-de34-48f5-9bbf-5debf5e14b77
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18016"
    uid: ab4d1f97-acc2-406b-9489-7a814755d284
  kind: Event
  lastTimestamp: "2026-07-28T01:42:27Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:41:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:42:27Z"
    name: isvc-logger-raw-18035-predictor.18c65098547b571b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18404"
    uid: 60ceced7-ba9d-46d3-8d42-b0c5c6454a8d
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:42:12Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18184"
    uid: ab4d1f97-acc2-406b-9489-7a814755d284
  kind: Event
  lastTimestamp: "2026-07-28T01:43:12Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:42:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:43:12Z"
    name: isvc-logger-raw-18035-predictor.18c6509bd38a0436
    namespace: kserve-ci-e2e-test
    resourceVersion: "18738"
    uid: 5ff0c47d-d662-4046-a945-634442a935e6
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:42:12Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18184"
    uid: ab4d1f97-acc2-406b-9489-7a814755d284
  kind: Event
  lastTimestamp: "2026-07-28T01:43:12Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:42:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:43:12Z"
    name: isvc-logger-raw-18035-predictor.18c6509bd3ea9c48
    namespace: kserve-ci-e2e-test
    resourceVersion: "18739"
    uid: a0b2b596-c4e5-42a4-9775-5652d0eaad66
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-18035
    namespace: kserve-ci-e2e-test
    resourceVersion: "18008"
    uid: 5e16f9e4-340d-4ad1-a429-19c465cd69f5
  kind: Event
  lastTimestamp: "2026-07-28T01:43:04Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:41:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:43:04Z"
    name: isvc-logger-raw-18035.18c65094d5918a78
    namespace: kserve-ci-e2e-test
    resourceVersion: "18680"
    uid: 875c19f6-7e2d-443b-a703-fba628f05c1d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-18035
    namespace: kserve-ci-e2e-test
    resourceVersion: "18008"
    uid: 5e16f9e4-340d-4ad1-a429-19c465cd69f5
  kind: Event
  lastTimestamp: "2026-07-28T01:41:42Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-18035":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-18035":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:41:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:42Z"
    name: isvc-logger-raw-18035.18c65094d621024f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18023"
    uid: 826221cb-d655-443a-bd06-4d5118101d50
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:04Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-18035
    namespace: kserve-ci-e2e-test
    resourceVersion: "18676"
    uid: 5e16f9e4-340d-4ad1-a429-19c465cd69f5
  kind: Event
  lastTimestamp: "2026-07-28T01:43:04Z"
  message: InferenceService [isvc-logger-raw-18035] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:43:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:43:04Z"
    name: isvc-logger-raw-18035.18c650a805850914
    namespace: kserve-ci-e2e-test
    resourceVersion: "18677"
    uid: bd13e76c-2d0b-4316-9249-ccc79fcf6f08
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:55:29.501829Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24758"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c6515564a8d75f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24762"
    uid: 8e68770e-734c-4145-b4d7-f16f7acf68bc
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24766"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:29Z"
  message: Add eth0 [10.134.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651557e2ad784
    namespace: kserve-ci-e2e-test
    resourceVersion: "24775"
    uid: 5e4a7519-9cef-4605-8187-9711422aa1ec
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:29Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651557f8c1c22
    namespace: kserve-ci-e2e-test
    resourceVersion: "24777"
    uid: 09063aa2-ce9e-4395-bde0-91a9458a5ec4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:30Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:55:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:30Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c6515582b07e9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "24778"
    uid: 5a374dc4-1926-46cf-bddd-844a8ab30681
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:30Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:55:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:30Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651558327b3ce
    namespace: kserve-ci-e2e-test
    resourceVersion: "24779"
    uid: 7ea8fd00-4059-4692-8e2f-816d0037b2c3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:34Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:55:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:34Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c65156988f3757
    namespace: kserve-ci-e2e-test
    resourceVersion: "24818"
    uid: 97455315-96b4-48d0-b8bb-ba13ca7c18e3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:34Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:55:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:34Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651569b86078f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24821"
    uid: 08917408-f64c-4681-b2e3-d0f08bb13282
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:34Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:55:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:34Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651569c07b161
    namespace: kserve-ci-e2e-test
    resourceVersion: "24822"
    uid: 389579c1-2638-45cf-8780-aa797f21d269
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:34Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:55:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:34Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651569c1391b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24823"
    uid: ee7f3440-12a8-48fe-b300-91a5d025dc03
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:34Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:55:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:34Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651569f0a7d05
    namespace: kserve-ci-e2e-test
    resourceVersion: "24826"
    uid: fe5833c4-6fac-44ff-94c7-dace50195ace
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:55:34Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:55:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:55:34Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c651569f80e337
    namespace: kserve-ci-e2e-test
    resourceVersion: "24827"
    uid: 1bd27d8d-98ce-4496-b5ae-705d87cc97ba
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:56:42Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:55:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:56:42Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c6515710579542
    namespace: kserve-ci-e2e-test
    resourceVersion: "25586"
    uid: 4fe09be3-6b0a-4714-ba32-851caa5a2202
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:57:20Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:20Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c6516f4ae06f63
    namespace: kserve-ci-e2e-test
    resourceVersion: "25950"
    uid: b5d07cd8-2c4d-499c-a8c6-7143221a903a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:57:20Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:20Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c6516f4ae0dd36
    namespace: kserve-ci-e2e-test
    resourceVersion: "25951"
    uid: 5a924eac-bb1c-46bd-b8b8-23498f3b9d4e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "24760"
    uid: 766b16e2-8e42-42a1-9fbd-3f781d4a70d9
  kind: Event
  lastTimestamp: "2026-07-28T01:57:22Z"
  message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp
    10.134.0.30:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:57:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:22Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7-8llcd.18c6516fbe9e957d
    namespace: kserve-ci-e2e-test
    resourceVersion: "25992"
    uid: 3aaa121b-fa56-4db6-bb9e-8c700a5667da
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-primary-733ea5-predictor-79b856fbf7
    namespace: kserve-ci-e2e-test
    resourceVersion: "24749"
    uid: c83f7c2d-2815-4917-bd30-9da7a0a739c3
  kind: Event
  lastTimestamp: "2026-07-28T01:55:29Z"
  message: 'Created pod: isvc-primary-733ea5-predictor-79b856fbf7-8llcd'
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5-predictor-79b856fbf7.18c6515564375684
    namespace: kserve-ci-e2e-test
    resourceVersion: "24759"
    uid: 63e9eb19-5595-422a-9c5a-be321fd33afc
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-primary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24746"
    uid: 33b3ff2d-2d09-49e4-89a7-2e2dd9010fd4
  kind: Event
  lastTimestamp: "2026-07-28T01:55:29Z"
  message: Scaled up replica set isvc-primary-733ea5-predictor-79b856fbf7 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5-predictor.18c651555f9cfddc
    namespace: kserve-ci-e2e-test
    resourceVersion: "24750"
    uid: 2f6cb723-edcd-4e37-b2e3-ab01f100ce80
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:44Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24748"
    uid: afb23010-2430-47c2-bf84-415f57db354c
  kind: Event
  lastTimestamp: "2026-07-28T01:56:14Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:56:14Z"
    name: isvc-primary-733ea5-predictor.18c65158debe7588
    namespace: kserve-ci-e2e-test
    resourceVersion: "25108"
    uid: 3f29d130-1e62-4f3d-b129-7f64cae7db49
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:44Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24748"
    uid: afb23010-2430-47c2-bf84-415f57db354c
  kind: Event
  lastTimestamp: "2026-07-28T01:56:14Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:56:14Z"
    name: isvc-primary-733ea5-predictor.18c65158df21e1a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "25109"
    uid: 4dc49a0d-a4c6-466e-b4bf-baa222025a2a
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:56:29Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24898"
    uid: afb23010-2430-47c2-bf84-415f57db354c
  kind: Event
  lastTimestamp: "2026-07-28T01:57:14Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:56:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:14Z"
    name: isvc-primary-733ea5-predictor.18c651635bb99e12
    namespace: kserve-ci-e2e-test
    resourceVersion: "25884"
    uid: a444c314-5cfb-406d-9822-f602e13a5ff9
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:56:29Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24898"
    uid: afb23010-2430-47c2-bf84-415f57db354c
  kind: Event
  lastTimestamp: "2026-07-28T01:57:14Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:56:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:14Z"
    name: isvc-primary-733ea5-predictor.18c651635c1801c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "25885"
    uid: 8d616dd0-c883-40aa-82c9-be3602ec7be0
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24739"
    uid: 89ef4f16-bc9d-48af-852c-43660db28c10
  kind: Event
  lastTimestamp: "2026-07-28T01:56:48Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:56:48Z"
    name: isvc-primary-733ea5.18c651555fa3f716
    namespace: kserve-ci-e2e-test
    resourceVersion: "25638"
    uid: 3f7c0e84-2de9-4ff3-9790-886f8cb6d3a2
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24739"
    uid: 89ef4f16-bc9d-48af-852c-43660db28c10
  kind: Event
  lastTimestamp: "2026-07-28T01:55:29Z"
  message: 'Failed to update status for InferenceService "isvc-primary-733ea5": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-733ea5":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5.18c65155603882d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "24753"
    uid: 179ab1a4-730e-4d3f-9489-16ad4d5975a7
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:55:29Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "24739"
    uid: 89ef4f16-bc9d-48af-852c-43660db28c10
  kind: Event
  lastTimestamp: "2026-07-28T01:55:29Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-primary-733ea5": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:55:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:55:29Z"
    name: isvc-primary-733ea5.18c65155603915b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "24755"
    uid: c884d16b-f93e-4463-a2a9-8b2b9f8a651d
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:56:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "25636"
    uid: 89ef4f16-bc9d-48af-852c-43660db28c10
  kind: Event
  lastTimestamp: "2026-07-28T01:56:48Z"
  message: InferenceService [isvc-primary-733ea5] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:56:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:56:48Z"
    name: isvc-primary-733ea5.18c65167ce23a41a
    namespace: kserve-ci-e2e-test
    resourceVersion: "25637"
    uid: 7101b4eb-2b3f-448f-8696-733cabda1e15
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:36:06.716533Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14956"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:36:06Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:36:06Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c65046a94180e3
    namespace: kserve-ci-e2e-test
    resourceVersion: "14960"
    uid: 95d568ce-7333-4372-90df-fbc869d7e8d6
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14966"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:07Z"
  message: Add eth0 [10.134.0.15/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:36:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:36:07Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c65046c2ab6c5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "14981"
    uid: bea13962-60c8-4243-863e-bbceb5bf162b
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:07Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
  metadata:
    creationTimestamp: "2026-07-28T01:36:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:07Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c65046c4332874
    namespace: kserve-ci-e2e-test
    resourceVersion: "14983"
    uid: fe8e6ab7-88b5-4891-ba15-38d4f488c87b
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:10Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    in 2.947s (2.947s including waiting). Image size: 300981723 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:36:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:10Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504773e14d3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15003"
    uid: 02645728-9729-4126-b040-77d2945c5498
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:10Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:36:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:10Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504776c3a2bd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15004"
    uid: 6641dfe1-d751-406f-9047-a4eaacdccc47
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:10Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:36:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:10Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c65047774095a8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15005"
    uid: 606bc063-02fd-4489-aefa-5c0a1909bc76
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:14Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
  metadata:
    creationTimestamp: "2026-07-28T01:36:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:14Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c650487817bb95
    namespace: kserve-ci-e2e-test
    resourceVersion: "15041"
    uid: 9e927c92-55e6-4a20-b652-1c18e09f9134
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:27Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    in 13.023s (13.023s including waiting). Image size: 1555730357 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:36:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:27Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504b806225a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15144"
    uid: b7c88983-8a8f-469f-a094-77151b349d6d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:27Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:36:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:27Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504b847ffeab
    namespace: kserve-ci-e2e-test
    resourceVersion: "15145"
    uid: 69a11874-33f4-481d-b39d-2cf42a3ef409
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:27Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:36:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:27Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504b84fa4450
    namespace: kserve-ci-e2e-test
    resourceVersion: "15147"
    uid: 436007b2-d8c2-4dab-b6b7-dc46c7c22d3e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:27Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-07-28T01:36:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:27Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504b85070618
    namespace: kserve-ci-e2e-test
    resourceVersion: "15148"
    uid: fc7ce61f-4cc5-43ee-b5de-050231405fed
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:29Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.267s (2.267s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:36:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:29Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c0c2cee37
    namespace: kserve-ci-e2e-test
    resourceVersion: "15164"
    uid: 349bc733-a919-4384-8a1d-960a3fea39db
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:29Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:36:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:29Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c0f0f455a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15165"
    uid: df012e8e-374c-48d9-8595-853d3941ffdc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:29Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:36:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:29Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c0f89caf3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15166"
    uid: 650fa465-ac00-4d94-a792-d239a63d341d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:29Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:bf42462edec7da298c1b007872394204ebe10a2c0c0d143725247988fb0a7324"
  metadata:
    creationTimestamp: "2026-07-28T01:36:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:29Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c0f96c57f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15167"
    uid: c1cb12d4-7cf6-48cd-984f-3dbf4277d48c
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:32Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:bf42462edec7da298c1b007872394204ebe10a2c0c0d143725247988fb0a7324"
    in 2.245s (2.245s including waiting). Image size: 240177278 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:36:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:32Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c9570e6ee
    namespace: kserve-ci-e2e-test
    resourceVersion: "15183"
    uid: af07af69-957d-4e6a-8724-ca10f282307d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:32Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-07-28T01:36:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:32Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c98b21cad
    namespace: kserve-ci-e2e-test
    resourceVersion: "15184"
    uid: 80269d3b-ffb9-46cc-ad18-7f32932f8555
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:36:32Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-07-28T01:36:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:36:32Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504c99201eb0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15185"
    uid: a24201ab-5cfe-4735-a987-62deded8a0ef
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:38:03Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.15:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:36:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:03Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504cadfef7cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "15977"
    uid: 4db62f83-9b5e-4e07-ba14-5ec3b9352977
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:38:03Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T01:36:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:03Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6504cae08d12e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15978"
    uid: ad55e0c3-ded1-4d9a-a9a5-48b9721b2432
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6505f22fa0b67
    namespace: kserve-ci-e2e-test
    resourceVersion: "15809"
    uid: 61c87626-0bcc-439e-9857-7bf320c18d2a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6505f22fac405
    namespace: kserve-ci-e2e-test
    resourceVersion: "15814"
    uid: b21a09f2-07b4-49e7-bf4b-ac809fc45885
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6505f22faf145
    namespace: kserve-ci-e2e-test
    resourceVersion: "15817"
    uid: 06b9f2ed-91e9-40d8-9b1d-2e96114036e5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw
    namespace: kserve-ci-e2e-test
    resourceVersion: "14958"
    uid: 23545a18-dca2-4741-bae5-e7ecaf395f63
  kind: Event
  lastTimestamp: "2026-07-28T01:38:03Z"
  message: 'Readiness probe failed: Get "https://10.134.0.15:8643/healthz": dial tcp
    10.134.0.15:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:37:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:03Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw.18c6505f8a288c4a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15980"
    uid: 0b117a38-51c5-4138-b763-292d4ae1e375
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14945"
    uid: f0b3f70b-66f3-4406-a3f4-3d12b836a216
  kind: Event
  lastTimestamp: "2026-07-28T01:36:06Z"
  message: 'Created pod: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b-8dmxw'
  metadata:
    creationTimestamp: "2026-07-28T01:36:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:36:06Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b.18c65046a8b5dab1
    namespace: kserve-ci-e2e-test
    resourceVersion: "14957"
    uid: 4c7f228f-5f21-4c1e-ae0e-6dad10a059c1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-raw-sklearn-batcher-a4889-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14944"
    uid: be010fc3-4db4-4f06-be54-7a0f1d81ca44
  kind: Event
  lastTimestamp: "2026-07-28T01:36:06Z"
  message: Scaled up replica set isvc-raw-sklearn-batcher-a4889-predictor-6c6f67fb9b
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:36:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:36:06Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor.18c65046a03d8d77
    namespace: kserve-ci-e2e-test
    resourceVersion: "14946"
    uid: bbb7785d-baf7-4c8e-b282-8e0f776e446c
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-a4889-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14952"
    uid: b8eba8bd-2384-477f-9d78-2ac6ae6b563f
  kind: Event
  lastTimestamp: "2026-07-28T01:37:06Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:36:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:06Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor.18c6504a256e2691
    namespace: kserve-ci-e2e-test
    resourceVersion: "15430"
    uid: e4290c30-0df9-4215-8dc0-46863202206b
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-a4889-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14952"
    uid: b8eba8bd-2384-477f-9d78-2ac6ae6b563f
  kind: Event
  lastTimestamp: "2026-07-28T01:37:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:36:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:06Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor.18c6504a25d0c981
    namespace: kserve-ci-e2e-test
    resourceVersion: "15431"
    uid: a9d2de16-0411-4cac-a6fa-c33353cea6e8
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-a4889-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15102"
    uid: b8eba8bd-2384-477f-9d78-2ac6ae6b563f
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:37:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor.18c6505821b3d792
    namespace: kserve-ci-e2e-test
    resourceVersion: "15781"
    uid: 5c572d18-d5a7-40e7-b20c-4f49bd67a7fb
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-a4889-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15102"
    uid: b8eba8bd-2384-477f-9d78-2ac6ae6b563f
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:37:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-raw-sklearn-batcher-a4889-predictor.18c65058222093ce
    namespace: kserve-ci-e2e-test
    resourceVersion: "15782"
    uid: 6f402b14-2419-4c71-800e-a6215fabcbc3
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-a4889
    namespace: kserve-ci-e2e-test
    resourceVersion: "14936"
    uid: 460ebc4c-9677-420a-ab53-5e3c8c25e3a1
  kind: Event
  lastTimestamp: "2026-07-28T01:37:43Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:36:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:37:43Z"
    name: isvc-raw-sklearn-batcher-a4889.18c65046a6716185
    namespace: kserve-ci-e2e-test
    resourceVersion: "15705"
    uid: 0e92bcda-a683-4459-94c0-96cd1dac64f6
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-a4889
    namespace: kserve-ci-e2e-test
    resourceVersion: "14936"
    uid: 460ebc4c-9677-420a-ab53-5e3c8c25e3a1
  kind: Event
  lastTimestamp: "2026-07-28T01:36:06Z"
  message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-a4889":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a4889":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:36:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:36:06Z"
    name: isvc-raw-sklearn-batcher-a4889.18c65046a6ec9495
    namespace: kserve-ci-e2e-test
    resourceVersion: "14970"
    uid: bf4c1ce6-1dde-48c5-94cf-9c9ef89dae06
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:36:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-a4889
    namespace: kserve-ci-e2e-test
    resourceVersion: "14936"
    uid: 460ebc4c-9677-420a-ab53-5e3c8c25e3a1
  kind: Event
  lastTimestamp: "2026-07-28T01:36:06Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a4889": the object
    has been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:36:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:36:06Z"
    name: isvc-raw-sklearn-batcher-a4889.18c65046a6ece81a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14972"
    uid: 98f37006-b8aa-4f89-99c5-7c37c3e89b56
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:43Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-a4889
    namespace: kserve-ci-e2e-test
    resourceVersion: "15702"
    uid: 460ebc4c-9677-420a-ab53-5e3c8c25e3a1
  kind: Event
  lastTimestamp: "2026-07-28T01:37:43Z"
  message: InferenceService [isvc-raw-sklearn-batcher-a4889] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:37:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:37:43Z"
    name: isvc-raw-sklearn-batcher-a4889.18c6505d345a499e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15703"
    uid: b9290752-ee3b-4526-9678-4a080b893c3e
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:57:04.820376Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    namespace: kserve-ci-e2e-test
    resourceVersion: "25780"
    uid: c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    to ip-10-0-135-95.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:57:04Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:57:04Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2.18c6516b96167c99
    namespace: kserve-ci-e2e-test
    resourceVersion: "25785"
    uid: 071f83aa-90f2-4f31-be67-1a034948bd9d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:05Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    namespace: kserve-ci-e2e-test
    resourceVersion: "25789"
    uid: c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676
  kind: Event
  lastTimestamp: "2026-07-28T01:57:05Z"
  message: Add eth0 [10.133.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:57:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:57:05Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2.18c6516baffc1e51
    namespace: kserve-ci-e2e-test
    resourceVersion: "25796"
    uid: 676405c6-3e5d-4fcd-8cbf-b32cae1964dc
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    namespace: kserve-ci-e2e-test
    resourceVersion: "25782"
    uid: c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676
  kind: Event
  lastTimestamp: "2026-07-28T01:57:08Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:57:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:08Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2.18c6516bb1863870
    namespace: kserve-ci-e2e-test
    resourceVersion: "25829"
    uid: c24490f9-0f7a-4e24-8a19-491830539827
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    namespace: kserve-ci-e2e-test
    resourceVersion: "25782"
    uid: c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676
  kind: Event
  lastTimestamp: "2026-07-28T01:57:08Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:57:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:08Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2.18c6516bb47d087f
    namespace: kserve-ci-e2e-test
    resourceVersion: "25833"
    uid: 56140609-74bd-4581-bd2c-7b8e98500314
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    namespace: kserve-ci-e2e-test
    resourceVersion: "25782"
    uid: c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676
  kind: Event
  lastTimestamp: "2026-07-28T01:57:08Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:57:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:08Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2.18c6516bb5057aaf
    namespace: kserve-ci-e2e-test
    resourceVersion: "25836"
    uid: 3d30f1e0-b896-4a2e-b588-d66e721a70df
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2
    namespace: kserve-ci-e2e-test
    resourceVersion: "25782"
    uid: c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676
  kind: Event
  lastTimestamp: "2026-07-28T01:57:11Z"
  message: Back-off restarting failed container storage-initializer in pod isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2_kserve-ci-e2e-test(c3ae9af5-f733-4bb9-b1f8-a6c7a06e9676)
  metadata:
    creationTimestamp: "2026-07-28T01:57:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:11Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2.18c6516d37316ec0
    namespace: kserve-ci-e2e-test
    resourceVersion: "25857"
    uid: 80e69da8-23e0-40a1-b709-13f57381226c
  reason: BackOff
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:04Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-secondary-733ea5-predictor-7959dcc5d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "25768"
    uid: e7c7834d-1885-4645-ae17-e49ce56ccead
  kind: Event
  lastTimestamp: "2026-07-28T01:57:04Z"
  message: 'Created pod: isvc-secondary-733ea5-predictor-7959dcc5d6-dwdz2'
  metadata:
    creationTimestamp: "2026-07-28T01:57:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:04Z"
    name: isvc-secondary-733ea5-predictor-7959dcc5d6.18c6516b955e250c
    namespace: kserve-ci-e2e-test
    resourceVersion: "25781"
    uid: 265e8b59-84bc-41e7-ab02-d6b18fdc3359
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:04Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-secondary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25766"
    uid: 90b70ce8-01fc-42d0-ba89-3982d662be27
  kind: Event
  lastTimestamp: "2026-07-28T01:57:04Z"
  message: Scaled up replica set isvc-secondary-733ea5-predictor-7959dcc5d6 from 0
    to 1
  metadata:
    creationTimestamp: "2026-07-28T01:57:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:04Z"
    name: isvc-secondary-733ea5-predictor.18c6516b8deeb11c
    namespace: kserve-ci-e2e-test
    resourceVersion: "25770"
    uid: ee301eb8-7917-4967-b32a-ceac30358ed7
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:19Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25769"
    uid: 2948dd8a-0a1f-4acb-b272-aa7f88f24706
  kind: Event
  lastTimestamp: "2026-07-28T01:57:19Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:57:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:19Z"
    name: isvc-secondary-733ea5-predictor.18c6516f0cc8d863
    namespace: kserve-ci-e2e-test
    resourceVersion: "25925"
    uid: 5d6dc726-0c73-48b3-bacd-85b49f2985a1
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:19Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-733ea5-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "25769"
    uid: 2948dd8a-0a1f-4acb-b272-aa7f88f24706
  kind: Event
  lastTimestamp: "2026-07-28T01:57:19Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:57:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:19Z"
    name: isvc-secondary-733ea5-predictor.18c6516f0d300b16
    namespace: kserve-ci-e2e-test
    resourceVersion: "25927"
    uid: c031b41f-5f2b-4296-9ae9-f81e4c8d20ae
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:04Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "25759"
    uid: 801a3871-93ec-45dd-b317-33f0f65fefe0
  kind: Event
  lastTimestamp: "2026-07-28T01:57:11Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:57:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:11Z"
    name: isvc-secondary-733ea5.18c6516b8e3c8c8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "25863"
    uid: d416cab2-b071-4a64-9b59-b097c190b198
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:04Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "25759"
    uid: 801a3871-93ec-45dd-b317-33f0f65fefe0
  kind: Event
  lastTimestamp: "2026-07-28T01:57:04Z"
  message: 'Failed to update status for InferenceService "isvc-secondary-733ea5":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-733ea5":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:57:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:04Z"
    name: isvc-secondary-733ea5.18c6516b8ec76e35
    namespace: kserve-ci-e2e-test
    resourceVersion: "25773"
    uid: 27ddc460-d153-4e51-b169-c034561be359
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:04Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-733ea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "25759"
    uid: 801a3871-93ec-45dd-b317-33f0f65fefe0
  kind: Event
  lastTimestamp: "2026-07-28T01:57:04Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-secondary-733ea5": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:57:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:04Z"
    name: isvc-secondary-733ea5.18c6516b8ec7d72d
    namespace: kserve-ci-e2e-test
    resourceVersion: "25774"
    uid: 35837d42-38f1-407d-9229-d3f704533718
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:37:51.837124Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15799"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6505f22ee3686
    namespace: kserve-ci-e2e-test
    resourceVersion: "15811"
    uid: e50b86c2-26ee-46d4-9042-f40c106c8ebe
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15829"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: Add eth0 [10.134.0.16/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6505f3cc47b10
    namespace: kserve-ci-e2e-test
    resourceVersion: "15856"
    uid: d0186988-e42f-404c-8808-6093925ca9d4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6505f3e68b437
    namespace: kserve-ci-e2e-test
    resourceVersion: "15858"
    uid: ade2278d-0c2b-4cf9-b308-de9d48000485
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6505f429417db
    namespace: kserve-ci-e2e-test
    resourceVersion: "15861"
    uid: e53b902b-55a9-41a1-a419-6bf78125e835
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6505f4363b30d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15863"
    uid: 40688f37-903c-44aa-95c8-7ef5b7ffb0be
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c650604e55333a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15910"
    uid: 15398863-4717-4306-a599-ee2760fe4599
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c65060510d5c69
    namespace: kserve-ci-e2e-test
    resourceVersion: "15914"
    uid: 90354aa9-1b99-45cc-8427-07d5466797f4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c650605178fa50
    namespace: kserve-ci-e2e-test
    resourceVersion: "15915"
    uid: fecc3ddd-f45a-4296-a53f-f5a7f03a9b8e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6506051833c19
    namespace: kserve-ci-e2e-test
    resourceVersion: "15916"
    uid: b3b17dd2-6093-439a-a45a-79c7c85d5f76
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6506053fc960e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15919"
    uid: 977d66d7-4367-4053-a1f2-5f859feabd02
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c65060548543b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15920"
    uid: f52a9ad9-7631-44e3-a7b8-9910a0ec7925
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:39:08Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.16:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:37:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:08Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c650608a729dad
    namespace: kserve-ci-e2e-test
    resourceVersion: "16477"
    uid: 53804498-6930-4666-85ed-3e9a3bea0f10
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c65078cf015813
    namespace: kserve-ci-e2e-test
    resourceVersion: "16824"
    uid: 7bd18222-ed66-4e6f-9538-d2b8c2041bea
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c65078cf01e760
    namespace: kserve-ci-e2e-test
    resourceVersion: "16827"
    uid: 0db268e1-07e4-44c2-9336-12c5ca812e19
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82
    namespace: kserve-ci-e2e-test
    resourceVersion: "15806"
    uid: 8ceffe1e-148f-4068-b1ab-71795c00c059
  kind: Event
  lastTimestamp: "2026-07-28T01:39:43Z"
  message: 'Readiness probe failed: Get "https://10.134.0.16:8643/healthz": dial tcp
    10.134.0.16:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:39:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:43Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82.18c6507938d3df41
    namespace: kserve-ci-e2e-test
    resourceVersion: "16924"
    uid: 65765f1f-9df9-4822-ac46-94a97092d99d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964
    namespace: kserve-ci-e2e-test
    resourceVersion: "15774"
    uid: 5da66298-b012-468b-b2a9-3c928078a387
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: 'Created pod: isvc-sklearn-graph-raw-b5420-predictor-6795699964-7vb82'
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-sklearn-graph-raw-b5420-predictor-6795699964.18c6505f2266fbb0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15807"
    uid: 78237135-2d07-4980-87ca-b441407628ff
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15772"
    uid: bba0007d-25f7-42dc-9935-b01105d6fef0
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-b5420-predictor-6795699964
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-sklearn-graph-raw-b5420-predictor.18c6505f1db948ab
    namespace: kserve-ci-e2e-test
    resourceVersion: "15775"
    uid: 1632204a-6122-4236-a2dd-9520118c0c83
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15773"
    uid: 3f8ecb2c-51fd-4f26-aec7-6ba2380b4187
  kind: Event
  lastTimestamp: "2026-07-28T01:38:21Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:38:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:38:21Z"
    name: isvc-sklearn-graph-raw-b5420-predictor.18c650629c3bcfb4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16118"
    uid: 3042ec02-d137-4bee-95c4-493b23c0939c
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15773"
    uid: 3f8ecb2c-51fd-4f26-aec7-6ba2380b4187
  kind: Event
  lastTimestamp: "2026-07-28T01:38:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:38:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:38:21Z"
    name: isvc-sklearn-graph-raw-b5420-predictor.18c650629ca3ad8a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16119"
    uid: 42a7b7c1-9a3a-417d-ba10-f91fdd065db1
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16005"
    uid: 3f8ecb2c-51fd-4f26-aec7-6ba2380b4187
  kind: Event
  lastTimestamp: "2026-07-28T01:39:36Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:38:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:36Z"
    name: isvc-sklearn-graph-raw-b5420-predictor.18c650699a8654de
    namespace: kserve-ci-e2e-test
    resourceVersion: "16727"
    uid: 3b559201-cffc-4bdb-a4d2-6942f78cf2e5
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16005"
    uid: 3f8ecb2c-51fd-4f26-aec7-6ba2380b4187
  kind: Event
  lastTimestamp: "2026-07-28T01:39:36Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:38:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:36Z"
    name: isvc-sklearn-graph-raw-b5420-predictor.18c650699ad24254
    namespace: kserve-ci-e2e-test
    resourceVersion: "16728"
    uid: 6e730cf4-919c-41da-bef5-534661c735ea
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "15761"
    uid: 052646c3-6f28-44f2-b3c1-5f3a01594507
  kind: Event
  lastTimestamp: "2026-07-28T01:39:18Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:18Z"
    name: isvc-sklearn-graph-raw-b5420.18c6505f1dbf639f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16557"
    uid: dd2be1e6-83c0-4ff2-98ac-6a6d344cfeba
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "15761"
    uid: 052646c3-6f28-44f2-b3c1-5f3a01594507
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-b5420":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-b5420":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-sklearn-graph-raw-b5420.18c6505f1e29fde1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15778"
    uid: 24e31266-d74a-4a39-be71-c2529c8c3d85
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:18Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "16555"
    uid: 052646c3-6f28-44f2-b3c1-5f3a01594507
  kind: Event
  lastTimestamp: "2026-07-28T01:39:18Z"
  message: InferenceService [isvc-sklearn-graph-raw-b5420] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:39:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:18Z"
    name: isvc-sklearn-graph-raw-b5420.18c6507351bb2aa7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16556"
    uid: 9452c718-8db2-4edc-bad1-6d19935af318
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:39:42.165997Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16835"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c65078d30b33d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16844"
    uid: 2a56af59-c5ef-4a16-aaae-eb98ef57310d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16860"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Add eth0 [10.134.0.18/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c65078ec47f3e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16898"
    uid: b1ce80b6-acf9-4c47-a9b1-e425f147c8be
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c65078edb39fff
    namespace: kserve-ci-e2e-test
    resourceVersion: "16900"
    uid: 87a38ef9-ff6d-491c-a136-ceda9e418007
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c65078f00be1a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16901"
    uid: 263b2148-2481-4e8c-a227-55d29653ce9f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c65078f0705ccd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16902"
    uid: d62353a8-1465-46fd-b9a2-896a60380e43
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:47Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:39:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:47Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a0459cda6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16963"
    uid: 8987abcc-4142-4204-bc7c-1699d8a836d1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:47Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:39:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:47Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a07ef19aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "16967"
    uid: f111c0d3-685a-4929-bde0-49a3b099499f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:47Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:39:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:47Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a085aaac4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16968"
    uid: 0bd01629-f631-4fc5-a5e8-2ca5621b72cc
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:47Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:39:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:47Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a0865da62
    namespace: kserve-ci-e2e-test
    resourceVersion: "16970"
    uid: 58f8fc49-8e54-4cd8-9f01-3a1eaa6454b2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:47Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:39:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:47Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a0c1623c5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16973"
    uid: 4914cd1d-fc12-4455-8549-700c6b34639c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:39:47Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:39:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:47Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a0c811f5c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16974"
    uid: 13bd50fe-2c55-44b6-86b3-6be1c7b261d6
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:41:35Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:39:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:35Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6507a7cab3863
    namespace: kserve-ci-e2e-test
    resourceVersion: "17939"
    uid: e64679d5-9203-4747-8b50-b07656e38ef5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6509286f8f16d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17883"
    uid: ff99cf94-e71d-42b6-bb6c-ddd99a9e67c2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c6509286f98390
    namespace: kserve-ci-e2e-test
    resourceVersion: "17885"
    uid: 2e16d9e4-aad2-4dfb-b18e-8c922c80e9ff
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16841"
    uid: 96b0f7bb-0b37-49df-8970-0e19faf64778
  kind: Event
  lastTimestamp: "2026-07-28T01:41:35Z"
  message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp
    10.134.0.18:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:41:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:35Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn.18c650932ae122cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17938"
    uid: de6252d8-5b73-4c4b-99e8-b625c25210b1
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88
    namespace: kserve-ci-e2e-test
    resourceVersion: "16811"
    uid: 0f55a244-b28d-4c35-a559-5a3a0d9e77b6
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: 'Created pod: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88-bkpmn'
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88.18c65078d238387f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16838"
    uid: 814a4f71-98ec-49c6-bda6-0456004d314b
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16804"
    uid: 266c396b-5ebd-46aa-8973-4404b3e4e1cd
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-hpa-cd9f7-predictor-58b8468b88
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor.18c65078cd5c8d22
    namespace: kserve-ci-e2e-test
    resourceVersion: "16815"
    uid: a222508c-09de-4e4f-8cd1-1d1bcf753eec
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16807"
    uid: e096851f-d333-409e-9c21-672c175b7514
  kind: Event
  lastTimestamp: "2026-07-28T01:40:27Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:39:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:40:27Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor.18c6507c4c0da0d8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17287"
    uid: 54d86531-55e9-41c4-a5e2-e9a80d7e14ca
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16807"
    uid: e096851f-d333-409e-9c21-672c175b7514
  kind: Event
  lastTimestamp: "2026-07-28T01:40:27Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:39:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:40:27Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor.18c6507c4c760d43
    namespace: kserve-ci-e2e-test
    resourceVersion: "17288"
    uid: 97e96b8c-4968-42a6-9696-5932ed0ac678
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:40:12Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17062"
    uid: e096851f-d333-409e-9c21-672c175b7514
  kind: Event
  lastTimestamp: "2026-07-28T01:41:27Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:40:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:27Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor.18c6507fcb41df3d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17773"
    uid: 48330d94-e67b-4c92-a9f3-32ee8f1ef4f2
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:40:12Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17062"
    uid: e096851f-d333-409e-9c21-672c175b7514
  kind: Event
  lastTimestamp: "2026-07-28T01:41:27Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:40:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:27Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7-predictor.18c6507fcba8d776
    namespace: kserve-ci-e2e-test
    resourceVersion: "17774"
    uid: 3822640a-7043-497b-83ef-528c1d3a5bb9
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16792"
    uid: 8bc8c701-1686-4ba6-b97f-c64857b18e4b
  kind: Event
  lastTimestamp: "2026-07-28T01:41:05Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:05Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7.18c65078cda5b335
    namespace: kserve-ci-e2e-test
    resourceVersion: "17582"
    uid: ddd0b298-51a6-4b36-9d9b-1981fffe622c
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16792"
    uid: 8bc8c701-1686-4ba6-b97f-c64857b18e4b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-cd9f7":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-cd9f7":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7.18c65078ce30fdd8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16819"
    uid: 968553b4-bfb9-46c9-ab3e-574acd89a804
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:05Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17580"
    uid: 8bc8c701-1686-4ba6-b97f-c64857b18e4b
  kind: Event
  lastTimestamp: "2026-07-28T01:41:05Z"
  message: InferenceService [isvc-sklearn-graph-raw-hpa-cd9f7] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:41:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:05Z"
    name: isvc-sklearn-graph-raw-hpa-cd9f7.18c6508c35af34b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17581"
    uid: 76bf09fb-22c2-467a-8a26-0dec91892c8e
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:43:17.794927Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18816"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ab078791cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18821"
    uid: c3e2a660-c593-4c56-b0d9-971627c0e10f
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:18Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18834"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:18Z"
  message: Add eth0 [10.134.0.22/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:43:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:43:18Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ab20e5021b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18851"
    uid: 0c5dd2ce-7980-48da-bbb1-60b72b35ad15
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:18Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:43:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:18Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ab224ed70a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18854"
    uid: 07b9acf6-0eb5-407e-b1e5-dbe702a33933
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:18Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:43:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:18Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ab24e996d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18855"
    uid: a3691d0a-f68b-44ed-b495-7eb47a367c34
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:18Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:43:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:18Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ab25a5c0c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18856"
    uid: 1326bd23-b42f-4a03-bcf5-8d0f1967ab7e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:22Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:43:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:22Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac05e0754f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18894"
    uid: 006ae88f-5c70-4fe9-85b2-44e1244e0c6c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:22Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:43:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:22Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac0acf49b8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18899"
    uid: 6293fc39-69b0-4e7f-a24d-eaf7ead5c570
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:22Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:43:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:22Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac0b49fba1
    namespace: kserve-ci-e2e-test
    resourceVersion: "18900"
    uid: 1d2ce795-bbb1-4875-ade5-16e1a0520b59
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:22Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:43:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:22Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac0b541d09
    namespace: kserve-ci-e2e-test
    resourceVersion: "18901"
    uid: 2d0bea6c-086a-41c5-a828-c1a0d7b74843
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:22Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:43:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:22Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac0e180571
    namespace: kserve-ci-e2e-test
    resourceVersion: "18906"
    uid: 64815fc8-5c8c-4f05-bd8c-19300af233d8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:43:22Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:43:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:22Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac0e8f2cf3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18907"
    uid: f7f8bcd9-d7bf-46cc-88b4-d2bfeb1784b7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:46:07Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:43:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:07Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650ac420e4363
    namespace: kserve-ci-e2e-test
    resourceVersion: "20181"
    uid: eea6de0d-295f-48e7-bad3-dd2ef65f94ee
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:45:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:45:57Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:45:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:45:57Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650d0505a2bd4
    namespace: kserve-ci-e2e-test
    resourceVersion: "20038"
    uid: 2d97ec7c-7442-45b5-ae3c-b52ed4eafa0c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:45:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:45:57Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:45:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:45:57Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650d0505b953b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20039"
    uid: a2171fbe-c0d0-45ac-9008-b30f964855e5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:45:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: 104ebc58-a752-4fdc-bea2-30aec6ddb112
  kind: Event
  lastTimestamp: "2026-07-28T01:46:04Z"
  message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp
    10.134.0.22:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:45:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:46:04Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj.18c650d0949405da
    namespace: kserve-ci-e2e-test
    resourceVersion: "20149"
    uid: 82fbd7ca-5eac-4216-a71c-583fb3386ab6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18794"
    uid: 83b4875c-60f7-4a8c-9785-6ac38ee48d3f
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: 'Created pod: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd-455rj'
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd.18c650ab071ee280
    namespace: kserve-ci-e2e-test
    resourceVersion: "18817"
    uid: c6927caa-6aaf-4fe4-993f-5948ed51ce62
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-scale-raw-6ab94-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18787"
    uid: f0fc919c-ceb6-4e98-9f3b-fe04caad28a0
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: Scaled up replica set isvc-sklearn-scale-raw-6ab94-predictor-65fc4b98fd
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor.18c650ab023bbe55
    namespace: kserve-ci-e2e-test
    resourceVersion: "18798"
    uid: a3f9aafe-cea5-452c-9d98-18684ad7a595
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:32Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6ab94-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18792"
    uid: f470d0f5-a497-41b2-a077-b8c21c614f42
  kind: Event
  lastTimestamp: "2026-07-28T01:43:32Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:43:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:43:32Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor.18c650ae80df6276
    namespace: kserve-ci-e2e-test
    resourceVersion: "18987"
    uid: 2a900ec3-147d-4b98-ad93-7f033cb2a423
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:32Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6ab94-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18792"
    uid: f470d0f5-a497-41b2-a077-b8c21c614f42
  kind: Event
  lastTimestamp: "2026-07-28T01:43:32Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:43:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:43:32Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor.18c650ae814d318d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18989"
    uid: a45a964b-f556-47e1-bdb0-721817e413bb
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:47Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6ab94-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18988"
    uid: f470d0f5-a497-41b2-a077-b8c21c614f42
  kind: Event
  lastTimestamp: "2026-07-28T01:45:47Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:43:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:45:47Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor.18c650b20067dffe
    namespace: kserve-ci-e2e-test
    resourceVersion: "19945"
    uid: 883851c3-8df6-4c33-9f83-023df918548c
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:47Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-6ab94-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18988"
    uid: f470d0f5-a497-41b2-a077-b8c21c614f42
  kind: Event
  lastTimestamp: "2026-07-28T01:45:47Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:43:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:45:47Z"
    name: isvc-sklearn-scale-raw-6ab94-predictor.18c650b200cda9dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19946"
    uid: 963be56f-84a8-4ba8-9399-aa443131df06
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-6ab94
    namespace: kserve-ci-e2e-test
    resourceVersion: "18776"
    uid: ec182011-8b05-4004-941d-3e04f213add7
  kind: Event
  lastTimestamp: "2026-07-28T01:45:57Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:45:57Z"
    name: isvc-sklearn-scale-raw-6ab94.18c650ab02583425
    namespace: kserve-ci-e2e-test
    resourceVersion: "20020"
    uid: 764e4e39-2a9c-490d-bef1-38e00917c365
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-6ab94
    namespace: kserve-ci-e2e-test
    resourceVersion: "18776"
    uid: ec182011-8b05-4004-941d-3e04f213add7
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-6ab94":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-6ab94":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: isvc-sklearn-scale-raw-6ab94.18c650ab02d34323
    namespace: kserve-ci-e2e-test
    resourceVersion: "18804"
    uid: 65ed3511-0db4-43e0-bb80-4305a892dbd3
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:45:57Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-6ab94
    namespace: kserve-ci-e2e-test
    resourceVersion: "20018"
    uid: ec182011-8b05-4004-941d-3e04f213add7
  kind: Event
  lastTimestamp: "2026-07-28T01:45:57Z"
  message: InferenceService [isvc-sklearn-scale-raw-6ab94] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:45:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:45:57Z"
    name: isvc-sklearn-scale-raw-6ab94.18c650d03863a5b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20019"
    uid: 8fbc0d27-2d3b-4284-9787-65b2f3770ba2
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:37:51.920929Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15834"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6505f27ecfe1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15841"
    uid: 781def23-3cf4-4f06-be07-4e3fb090fee1
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15846"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: Add eth0 [10.134.0.17/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6505f41b0aa79
    namespace: kserve-ci-e2e-test
    resourceVersion: "15859"
    uid: 4af270a3-ea2f-44c9-af5c-b32739193198
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6505f4316046f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15862"
    uid: 339cd7fd-e20d-4cac-98bd-1849d085bd25
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6505f4607670b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15864"
    uid: dd198c20-04e9-4395-9c80-d259afd209e4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:37:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:37:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:52Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6505f4673e16a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15865"
    uid: b2efd81c-d09a-445d-80da-f886c187c176
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:37:56Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-07-28T01:37:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:37:56Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c650604e464bf9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15909"
    uid: 03958a24-da85-4289-8d20-8322c26b65c9
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:38:14Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.018s (18.018s
    including waiting). Image size: 1307355896 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:38:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:14Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c65064804688db
    namespace: kserve-ci-e2e-test
    resourceVersion: "16066"
    uid: 8b449f6e-cc39-427f-8644-bda2c4e6a791
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:38:14Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:38:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:14Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c65064837a136f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16068"
    uid: 5a0e6c84-5d55-4f94-bf27-b9c4d7d8d7f3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:38:14Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:38:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:14Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6506483f34eca
    namespace: kserve-ci-e2e-test
    resourceVersion: "16069"
    uid: a86a9e3d-26c4-406f-82e4-b2580dbbdb0b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:38:14Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:38:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:14Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6506483fd9c39
    namespace: kserve-ci-e2e-test
    resourceVersion: "16070"
    uid: c5d23b8a-2ac5-4ccf-be46-def8140c1d8b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:38:14Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:38:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:14Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c6506486d455a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16071"
    uid: 6f1f9838-c04c-46ee-b75f-c16442124ed1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:38:14Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:38:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:38:14Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c65064874de01e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16072"
    uid: 831417a2-740d-4507-a029-c31913793ab7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:39:11Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:38:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:11Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c65064c01a0278
    namespace: kserve-ci-e2e-test
    resourceVersion: "16499"
    uid: a26a978b-4c90-4a81-9879-6e9f3a52a435
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c65078dbab1850
    namespace: kserve-ci-e2e-test
    resourceVersion: "16889"
    uid: 2b154619-b504-4795-b7e9-a1a27aca8c83
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw
    namespace: kserve-ci-e2e-test
    resourceVersion: "15837"
    uid: 6ac07cff-0fe8-442c-af45-36191e5359d3
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw.18c65078dbabc131
    namespace: kserve-ci-e2e-test
    resourceVersion: "16890"
    uid: 84d93664-3885-429d-add9-69852ddaae95
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885
    namespace: kserve-ci-e2e-test
    resourceVersion: "15813"
    uid: dd9fcb70-ec40-4b26-b2da-9fc76f3bbf08
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: 'Created pod: isvc-xgboost-graph-raw-b5420-predictor-65476d6885-h8jkw'
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-xgboost-graph-raw-b5420-predictor-65476d6885.18c6505f2756bcb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15836"
    uid: 2bcbb765-9873-4454-8114-cba823efee56
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15805"
    uid: e3a1e9f3-7aab-405d-a468-98501b8c8a1e
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-b5420-predictor-65476d6885
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-xgboost-graph-raw-b5420-predictor.18c6505f23a881e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15816"
    uid: f47ac33f-3b8b-4d93-b059-f0cf39a78d8b
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15810"
    uid: c08f7c7c-60f5-4208-ad14-abc016cc6f7f
  kind: Event
  lastTimestamp: "2026-07-28T01:38:51Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:38:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:38:51Z"
    name: isvc-xgboost-graph-raw-b5420-predictor.18c65062a259886a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16347"
    uid: 5fd832dd-ea75-479b-a0f6-6e0be3799ad9
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:38:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15810"
    uid: c08f7c7c-60f5-4208-ad14-abc016cc6f7f
  kind: Event
  lastTimestamp: "2026-07-28T01:38:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:38:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:38:51Z"
    name: isvc-xgboost-graph-raw-b5420-predictor.18c65062a2be693b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16348"
    uid: 4cc8bde3-ccc4-440d-8ac8-a232e2bb2696
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16008"
    uid: c08f7c7c-60f5-4208-ad14-abc016cc6f7f
  kind: Event
  lastTimestamp: "2026-07-28T01:39:36Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:39:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:36Z"
    name: isvc-xgboost-graph-raw-b5420-predictor.18c650709d70ff16
    namespace: kserve-ci-e2e-test
    resourceVersion: "16730"
    uid: f6d36a30-cdd6-46ae-aee6-9c76d919e881
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-b5420-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16008"
    uid: c08f7c7c-60f5-4208-ad14-abc016cc6f7f
  kind: Event
  lastTimestamp: "2026-07-28T01:39:36Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:39:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:36Z"
    name: isvc-xgboost-graph-raw-b5420-predictor.18c650709dd964d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16731"
    uid: 30e95574-7a6c-4e2d-b688-675edf57d9f9
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "15780"
    uid: 632860f7-36e8-4ef9-97a3-c69a5be80ac0
  kind: Event
  lastTimestamp: "2026-07-28T01:39:22Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:22Z"
    name: isvc-xgboost-graph-raw-b5420.18c6505f23f0a82b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16592"
    uid: 9f35556f-a304-4693-b85f-d7d19d0a8106
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:37:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "15780"
    uid: 632860f7-36e8-4ef9-97a3-c69a5be80ac0
  kind: Event
  lastTimestamp: "2026-07-28T01:37:51Z"
  message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-b5420":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-b5420":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:37:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:37:51Z"
    name: isvc-xgboost-graph-raw-b5420.18c6505f2489cc85
    namespace: kserve-ci-e2e-test
    resourceVersion: "15825"
    uid: 4cfd2b91-1be0-4938-8c55-61a12bae5e3d
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:22Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "16589"
    uid: 632860f7-36e8-4ef9-97a3-c69a5be80ac0
  kind: Event
  lastTimestamp: "2026-07-28T01:39:22Z"
  message: InferenceService [isvc-xgboost-graph-raw-b5420] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:39:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:22Z"
    name: isvc-xgboost-graph-raw-b5420.18c650742416a6b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16590"
    uid: fc60d0ae-3144-4cb0-a502-e8a8fa9922c1
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:39:42.245796Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16861"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c65078d7cce0cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "16866"
    uid: 24f8c2f7-8817-40a8-a444-252c3f881cc3
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-cd9f7-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:43Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c65078e6af56c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16906"
    uid: 1b095159-8945-4e9a-8b41-9b49c85d54a2
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:44Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16875"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:44Z"
  message: Add eth0 [10.134.0.19/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:39:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:39:44Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c650794aa7e2ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "16926"
    uid: 6e20cb81-2c0a-4006-b413-16b6bb6976df
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:44Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:39:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:44Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c650794bfe9401
    namespace: kserve-ci-e2e-test
    resourceVersion: "16928"
    uid: cc39c494-b716-4bd4-9476-71059529d92c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:44Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:39:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:44Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c650794f6c69bd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16931"
    uid: cd5be057-40f6-479c-8cfb-53c50be1f360
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:44Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:39:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:44Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c650794fd48119
    namespace: kserve-ci-e2e-test
    resourceVersion: "16932"
    uid: 473b895f-bd9c-4a75-a67d-dad76f4a3cae
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:48Z"
  message: Container image "kserve/xgbserver:latest" already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:39:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:48Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507a40a3707a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16983"
    uid: 5762e668-eacb-4cf3-a1b0-d871ac87c3d7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:48Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:39:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:48Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507a4357e4a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16986"
    uid: 9903e336-a59a-463f-9038-dce97582f49a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:48Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:39:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:48Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507a43c37767
    namespace: kserve-ci-e2e-test
    resourceVersion: "16987"
    uid: fae285ad-441c-4cad-84a0-7b0bcb473fac
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:48Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:39:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:48Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507a43ceaab8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16988"
    uid: 727abc6f-3833-4d24-9bb3-b125b8cc3322
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:48Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:39:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:48Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507a46b90741
    namespace: kserve-ci-e2e-test
    resourceVersion: "16991"
    uid: b749e1d9-4116-4fcc-86f5-930d20276c21
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:39:48Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:39:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:48Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507a47290f56
    namespace: kserve-ci-e2e-test
    resourceVersion: "16992"
    uid: 9142e7f8-49df-4bc0-99f8-07e0fe4c0077
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:41:36Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:39:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:36Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6507ab875192e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17946"
    uid: 4be2f428-1b76-447b-97a2-394b0813a004
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6509289850420
    namespace: kserve-ci-e2e-test
    resourceVersion: "17900"
    uid: 8e3ba9af-46df-45dd-80bf-61c2cfed6bab
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6509289858678
    namespace: kserve-ci-e2e-test
    resourceVersion: "17902"
    uid: ef81a7ce-a17a-4abb-a435-9d4cd0e379d4
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 1915c51c-c201-4148-97fb-f5b2b703171b
  kind: Event
  lastTimestamp: "2026-07-28T01:41:36Z"
  message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp
    10.134.0.19:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:41:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:36Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4.18c6509366c2bcbd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17945"
    uid: e2c7df56-73d4-4f62-9ac7-cbc3feb611e4
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449
    namespace: kserve-ci-e2e-test
    resourceVersion: "16847"
    uid: 850b6cc3-ad7d-4091-91d4-ff73615778b0
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: 'Created pod: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449-8fqj4'
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449.18c65078d7555379
    namespace: kserve-ci-e2e-test
    resourceVersion: "16864"
    uid: 09b15f3c-314a-42d6-96b1-de862fb12c0c
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16843"
    uid: 01f3d1d5-2f64-4819-ab3e-6f7706f73ad3
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-hpa-cd9f7-predictor-69c7b46449
    from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor.18c65078d3ce39df
    namespace: kserve-ci-e2e-test
    resourceVersion: "16850"
    uid: 4ac66a60-78cf-4027-9cb2-a5a6df4b8fbe
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16845"
    uid: 2b97bc13-f164-4525-9127-87da2ed67556
  kind: Event
  lastTimestamp: "2026-07-28T01:40:27Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:39:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:40:27Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor.18c6507c5245a68c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17289"
    uid: c6674068-1944-4282-b296-c2d253ec0c90
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:57Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16845"
    uid: 2b97bc13-f164-4525-9127-87da2ed67556
  kind: Event
  lastTimestamp: "2026-07-28T01:40:27Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:39:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:40:27Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor.18c6507c52aebb18
    namespace: kserve-ci-e2e-test
    resourceVersion: "17290"
    uid: dfa49d83-29a8-4165-a968-97a6778de7c3
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:40:42Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17065"
    uid: 2b97bc13-f164-4525-9127-87da2ed67556
  kind: Event
  lastTimestamp: "2026-07-28T01:41:27Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:40:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:27Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor.18c65086cebb3ec4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17775"
    uid: e84bf629-103b-4d96-b800-a0670e76e446
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-07-28T01:40:42Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17065"
    uid: 2b97bc13-f164-4525-9127-87da2ed67556
  kind: Event
  lastTimestamp: "2026-07-28T01:41:27Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:40:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:27Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7-predictor.18c65086cf1e8dda
    namespace: kserve-ci-e2e-test
    resourceVersion: "17776"
    uid: 817b104b-68d3-421f-8daf-feb674ba65f3
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16829"
    uid: 284c237b-2429-47f3-b5c8-d38dbeb3e2fd
  kind: Event
  lastTimestamp: "2026-07-28T01:40:46Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:40:46Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7.18c65078d3fa3169
    namespace: kserve-ci-e2e-test
    resourceVersion: "17440"
    uid: d9807718-5236-4c87-a530-10dc43a725da
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16829"
    uid: 284c237b-2429-47f3-b5c8-d38dbeb3e2fd
  kind: Event
  lastTimestamp: "2026-07-28T01:39:42Z"
  message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-cd9f7":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-cd9f7":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:39:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:42Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7.18c65078d48f4c6e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16857"
    uid: d845a1ae-2327-49cf-a431-66828c24917d
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:40:46Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17436"
    uid: 284c237b-2429-47f3-b5c8-d38dbeb3e2fd
  kind: Event
  lastTimestamp: "2026-07-28T01:40:46Z"
  message: InferenceService [isvc-xgboost-graph-raw-hpa-cd9f7] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:40:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:40:46Z"
    name: isvc-xgboost-graph-raw-hpa-cd9f7.18c65087c81f5d7f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17438"
    uid: d1b2003d-200d-47cd-bf02-b1e46524a1c4
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:41:32.495485Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17851"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-18035-predictor-5fdc678c5-z569k
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c6509283319e35
    namespace: kserve-ci-e2e-test
    resourceVersion: "17860"
    uid: 52d5ab41-dd3d-41e8-a589-352faffdae9a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17877"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Add eth0 [10.134.0.20/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c650929cd3d9d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17909"
    uid: 1f290a72-21fb-4db9-8b38-094b7541cc69
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c650929e4e0c70
    namespace: kserve-ci-e2e-test
    resourceVersion: "17911"
    uid: d0c8efcd-57d7-49c1-9f61-176f4734376a
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:34Z"
  message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    in 1.313s (1.313s including waiting). Image size: 14813193 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:34Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c65092ec9e5944
    namespace: kserve-ci-e2e-test
    resourceVersion: "17920"
    uid: a117cf05-00b0-405b-9be1-ac9d2e2df369
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:34Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:34Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c65092ef7a893b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17921"
    uid: 5e8a4421-4a89-4614-9ec0-e3e086278aec
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:34Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:34Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c65092efefb9e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17922"
    uid: be0f8475-161c-4d80-8cb0-7bca7dcf8f5f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:34Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:34Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c65092eff9ae90
    namespace: kserve-ci-e2e-test
    resourceVersion: "17923"
    uid: 42e69236-2f03-4c56-92bb-c0e62587260d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:34Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:34Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c65092f2b64f87
    namespace: kserve-ci-e2e-test
    resourceVersion: "17925"
    uid: 8a5bb4e6-aec6-438b-8c0b-9425123d400f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:41:34Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:34Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c65092f32a682f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17926"
    uid: f6c83f70-73ad-426c-8039-6091c70f9e19
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c650ab0a366a09
    namespace: kserve-ci-e2e-test
    resourceVersion: "18840"
    uid: f7d6770e-de84-464e-96a2-1573da905639
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:43:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k
    namespace: kserve-ci-e2e-test
    resourceVersion: "17855"
    uid: 70bedd5b-7206-41fe-b0a0-f19456abe21f
  kind: Event
  lastTimestamp: "2026-07-28T01:43:17Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:43:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:43:17Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5-z569k.18c650ab0a36dbd2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18841"
    uid: dc8ba416-d962-4a4c-9103-7e76d90753ef
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-18035-predictor-5fdc678c5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17844"
    uid: c00334c3-c8c6-4cc4-9237-0555f99a1c78
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: 'Created pod: message-dumper-raw-18035-predictor-5fdc678c5-z569k'
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: message-dumper-raw-18035-predictor-5fdc678c5.18c65092826e4320
    namespace: kserve-ci-e2e-test
    resourceVersion: "17854"
    uid: 7a5cea81-8ff4-4956-829c-5571bda22d51
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17842"
    uid: 1db11698-bf4c-447f-b08e-e9a9d5b7950a
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Scaled up replica set message-dumper-raw-18035-predictor-5fdc678c5 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: message-dumper-raw-18035-predictor.18c650927e90dafa
    namespace: kserve-ci-e2e-test
    resourceVersion: "17845"
    uid: 7bfa5319-6257-4166-bcd2-3602de2e14d0
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17843"
    uid: 72ba3ef6-8601-421c-9c77-86d317d1d561
  kind: Event
  lastTimestamp: "2026-07-28T01:42:02Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:42:02Z"
    name: message-dumper-raw-18035-predictor.18c65095fdb08391
    namespace: kserve-ci-e2e-test
    resourceVersion: "18219"
    uid: b7eb254e-135a-4a99-b1b6-241bcf32c3ad
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:47Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-18035-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17843"
    uid: 72ba3ef6-8601-421c-9c77-86d317d1d561
  kind: Event
  lastTimestamp: "2026-07-28T01:42:02Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:41:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:42:02Z"
    name: message-dumper-raw-18035-predictor.18c65095fe084802
    namespace: kserve-ci-e2e-test
    resourceVersion: "18220"
    uid: 1767238c-b480-4cff-9b9e-cb6d0ab81ad4
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-18035
    namespace: kserve-ci-e2e-test
    resourceVersion: "17834"
    uid: e49c5ced-85bb-4b97-9bf7-b759d3cbcbfe
  kind: Event
  lastTimestamp: "2026-07-28T01:41:41Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:41Z"
    name: message-dumper-raw-18035.18c650927e9c1e64
    namespace: kserve-ci-e2e-test
    resourceVersion: "18000"
    uid: 31370c66-5e35-4976-bf86-599258324d88
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-18035
    namespace: kserve-ci-e2e-test
    resourceVersion: "17834"
    uid: e49c5ced-85bb-4b97-9bf7-b759d3cbcbfe
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: 'Failed to update status for InferenceService "message-dumper-raw-18035":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-18035":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: message-dumper-raw-18035.18c650927f1faf53
    namespace: kserve-ci-e2e-test
    resourceVersion: "17848"
    uid: c1eabaee-aa39-4663-8c24-9d694b9d91f6
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-18035
    namespace: kserve-ci-e2e-test
    resourceVersion: "17997"
    uid: e49c5ced-85bb-4b97-9bf7-b759d3cbcbfe
  kind: Event
  lastTimestamp: "2026-07-28T01:41:41Z"
  message: InferenceService [message-dumper-raw-18035] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:41:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:41Z"
    name: message-dumper-raw-18035.18c65094ac672421
    namespace: kserve-ci-e2e-test
    resourceVersion: "17998"
    uid: 763aa210-601f-451d-9e97-2747b3a20389
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:39:31.869380Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16670"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-b5420-6978dfb8f7-sb7sp
    to ip-10-0-143-186.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:39:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:39:31Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c650766d514dc1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16675"
    uid: 64a77f7d-6cd0-4705-8a41-b9394f881f71
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:32Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-b5420-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:32Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c650767a9f79a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16686"
    uid: 2cccaaea-a179-478b-b6a7-b9aa3ee0f4d5
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16678"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:32Z"
  message: Add eth0 [10.132.0.37/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:39:32Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c65076aac43376
    namespace: kserve-ci-e2e-test
    resourceVersion: "16695"
    uid: feebaa94-3255-4bb5-ae6f-1a6bb282c4e1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-b5420}
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:32Z"
  message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1"
  metadata:
    creationTimestamp: "2026-07-28T01:39:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:32Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c65076ac09e067
    namespace: kserve-ci-e2e-test
    resourceVersion: "16697"
    uid: ac715817-466b-4fd4-bf46-afa2e1c307ea
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-b5420}
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:34Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1"
    in 1.725s (1.725s including waiting). Image size: 218232445 bytes.'
  metadata:
    creationTimestamp: "2026-07-28T01:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:34Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c6507712e0d512
    namespace: kserve-ci-e2e-test
    resourceVersion: "16708"
    uid: 5cef6f8a-2573-4193-be99-94a8fdaf764c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-b5420}
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:34Z"
  message: 'Created container: model-chainer-raw-b5420'
  metadata:
    creationTimestamp: "2026-07-28T01:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:34Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c6507716b743df
    namespace: kserve-ci-e2e-test
    resourceVersion: "16709"
    uid: 595c3bf7-c09c-4790-b615-c4e01c987645
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-b5420}
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:34Z"
  message: Started container model-chainer-raw-b5420
  metadata:
    creationTimestamp: "2026-07-28T01:39:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:34Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c650771727c7c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16710"
    uid: 95567392-ad2b-47f0-93a7-8bd35b6dcf99
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-b5420}
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:39:41Z"
  message: Stopping container model-chainer-raw-b5420
  metadata:
    creationTimestamp: "2026-07-28T01:39:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:39:41Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c65078c3c7cd25
    namespace: kserve-ci-e2e-test
    resourceVersion: "16786"
    uid: 63b3a43c-5082-425c-8ecf-c180bf3589e3
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-b5420}
    kind: Pod
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp
    namespace: kserve-ci-e2e-test
    resourceVersion: "16673"
    uid: baea18be-28fe-4026-9c81-dc3046df4a57
  kind: Event
  lastTimestamp: "2026-07-28T01:40:10Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T01:39:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:40:10Z"
    name: model-chainer-raw-b5420-6978dfb8f7-sb7sp.18c65079af085b0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17164"
    uid: 6b2bfbd8-d491-4bdb-be5f-caedfaf35f76
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-b5420-6978dfb8f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16666"
    uid: feedca85-e189-4525-8e75-7c0181527956
  kind: Event
  lastTimestamp: "2026-07-28T01:39:31Z"
  message: 'Created pod: model-chainer-raw-b5420-6978dfb8f7-sb7sp'
  metadata:
    creationTimestamp: "2026-07-28T01:39:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:31Z"
    name: model-chainer-raw-b5420-6978dfb8f7.18c650766ce5e3c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16672"
    uid: 1b66a9af-ccd0-48e9-9836-87f3a71e2d60
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "16664"
    uid: 9d9dbeaf-6dd6-4000-a312-398d6e74d4ec
  kind: Event
  lastTimestamp: "2026-07-28T01:39:31Z"
  message: Scaled up replica set model-chainer-raw-b5420-6978dfb8f7 from 0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:39:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:39:31Z"
    name: model-chainer-raw-b5420.18c650766bdf8316
    namespace: kserve-ci-e2e-test
    resourceVersion: "16667"
    uid: cd32d327-9060-4a50-9956-8338e920ae01
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:39:31Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-b5420
    namespace: kserve-ci-e2e-test
    resourceVersion: "16681"
    uid: 653392d8-272d-49d3-9e3a-a3024ef8c858
  kind: Event
  lastTimestamp: "2026-07-28T01:39:31Z"
  message: InferenceGraph [model-chainer-raw-b5420] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:39:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:39:31Z"
    name: model-chainer-raw-b5420.18c65076719e8d5d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16683"
    uid: 473bbe4c-3641-4aa7-9408-fb444f074aaa
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:41:22.182559Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17708"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    to ip-10-0-143-186.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:41:22Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:41:22Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c650901c7ec707
    namespace: kserve-ci-e2e-test
    resourceVersion: "17713"
    uid: 58d1ec21-e625-4016-8321-6715588f221a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:22Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17710"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:22Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-cd9f7-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:41:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:22Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c6509024c51539
    namespace: kserve-ci-e2e-test
    resourceVersion: "17723"
    uid: 3dc15d78-ada3-4334-9fae-63c5c233a144
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:23Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17716"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:23Z"
  message: Add eth0 [10.132.0.38/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:41:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:41:23Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c6509059d42c28
    namespace: kserve-ci-e2e-test
    resourceVersion: "17732"
    uid: 4cf8b2d8-916f-4c92-b7e3-6ad7412febeb
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-cd9f7}
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17710"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:23Z"
  message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:41:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:23Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c650905b1e957f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17734"
    uid: 3aa2876c-783b-43f5-b561-d030b1d77bff
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-cd9f7}
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17710"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:23Z"
  message: 'Created container: model-chainer-raw-hpa-cd9f7'
  metadata:
    creationTimestamp: "2026-07-28T01:41:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:23Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c650905e36a5e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17735"
    uid: a7955ae3-bf9a-41da-b25c-6b79f223db06
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-cd9f7}
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17710"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:23Z"
  message: Started container model-chainer-raw-hpa-cd9f7
  metadata:
    creationTimestamp: "2026-07-28T01:41:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:23Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c650905ebd3cde
    namespace: kserve-ci-e2e-test
    resourceVersion: "17737"
    uid: e68175d1-3ce2-4a30-99e6-e79dd6a40276
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-cd9f7}
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17710"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:32Z"
  message: Stopping container model-chainer-raw-hpa-cd9f7
  metadata:
    creationTimestamp: "2026-07-28T01:41:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:32Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c6509273c8c86e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17829"
    uid: bede65b2-fddc-47c6-b629-7c2a6a00c6b2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-cd9f7}
    kind: Pod
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17710"
    uid: 9ed5802c-51d4-4a5b-82aa-eb8399a9746e
  kind: Event
  lastTimestamp: "2026-07-28T01:41:59Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-07-28T01:41:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:41:59Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc.18c65092f27b2a46
    namespace: kserve-ci-e2e-test
    resourceVersion: "18198"
    uid: 4f4553ce-b43e-4da8-bfe7-2d1a230f407e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-186.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-186.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:22Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-hpa-cd9f7-547498d659
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: ccf9a983-1b12-4144-b47f-bed111476527
  kind: Event
  lastTimestamp: "2026-07-28T01:41:22Z"
  message: 'Created pod: model-chainer-raw-hpa-cd9f7-547498d659-2qrnc'
  metadata:
    creationTimestamp: "2026-07-28T01:41:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:22Z"
    name: model-chainer-raw-hpa-cd9f7-547498d659.18c650901c1cafc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17711"
    uid: 79372575-47a6-4027-9049-95183600edd9
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:22Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17703"
    uid: 603d9181-3139-44a9-91ae-278118dc98d2
  kind: Event
  lastTimestamp: "2026-07-28T01:41:22Z"
  message: Scaled up replica set model-chainer-raw-hpa-cd9f7-547498d659 from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T01:41:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:41:22Z"
    name: model-chainer-raw-hpa-cd9f7.18c650901af6293e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17706"
    uid: 74b1b299-8943-4b91-a631-c0c7fb83181f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:41:22Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-cd9f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17719"
    uid: ef7cecc7-fc29-40e9-9daa-8e0604ae66ad
  kind: Event
  lastTimestamp: "2026-07-28T01:41:22Z"
  message: InferenceGraph [model-chainer-raw-hpa-cd9f7] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:41:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:41:22Z"
    name: model-chainer-raw-hpa-cd9f7.18c65090213c91c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17721"
    uid: 265bd558-7fe8-449e-a175-3a8ae9664602
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:57:36.212421Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26195"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    to ip-10-0-135-95.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:57:36Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:57:36Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65172e53346b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "26201"
    uid: 05a6d330-7373-4d8b-8a0c-bc05c0f689c9
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:36Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:36Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-9f304-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:36Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65172ef4cd8ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "26208"
    uid: 7f5e96ef-ef92-409b-a438-d08f94a260ad
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:37Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26205"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:37Z"
  message: Add eth0 [10.133.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:57:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:57:37Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c6517322764e49
    namespace: kserve-ci-e2e-test
    resourceVersion: "26222"
    uid: aa1875a9-9e2b-4734-800f-3e2d8527d141
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:37Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:57:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:37Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65173240856a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "26224"
    uid: 045f461b-d02f-4453-8625-c7e48dd9cd35
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:37Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:57:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:37Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65173273276f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "26226"
    uid: 1a0565ce-d052-4aa4-aff6-9d09022efde8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:37Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:57:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:37Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c6517327a2d477
    namespace: kserve-ci-e2e-test
    resourceVersion: "26227"
    uid: bb777b95-ffbe-4365-bcc1-c76863fba3b5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:40Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:40Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65173fe04f353
    namespace: kserve-ci-e2e-test
    resourceVersion: "26258"
    uid: 371524c7-fd20-406b-a250-64d9502d5004
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:40Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:40Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c6517400963ad1
    namespace: kserve-ci-e2e-test
    resourceVersion: "26261"
    uid: 18ed9ad2-a397-4abd-bc6e-8eac8e8065fd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:40Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:40Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c6517401013f71
    namespace: kserve-ci-e2e-test
    resourceVersion: "26262"
    uid: 69e810c8-93e3-4a1e-b9ac-aeb5480c3239
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:40Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:40Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65174010a9ab4
    namespace: kserve-ci-e2e-test
    resourceVersion: "26263"
    uid: 80bd5b15-59f8-4b90-9ddf-2e6bfab93430
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:41Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:57:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:41Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c6517403bd8d61
    namespace: kserve-ci-e2e-test
    resourceVersion: "26267"
    uid: 65f6dae8-0d00-4ca8-887b-1439eff248b2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:57:41Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:57:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:57:41Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65174042eeef1
    namespace: kserve-ci-e2e-test
    resourceVersion: "26268"
    uid: 969cc915-41b0-46ad-81d3-1ca22ce4c7ff
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:59:07Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:57:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:07Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c6517439fededd
    namespace: kserve-ci-e2e-test
    resourceVersion: "26908"
    uid: 7f9af984-d566-4426-93ae-c1c6b02c4271
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:59:06Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:59:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:06Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65187e1e18083
    namespace: kserve-ci-e2e-test
    resourceVersion: "26896"
    uid: e8483a4c-e123-46f8-b199-a25de9792d0d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:59:06Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:59:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:06Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c65187e1e2914a
    namespace: kserve-ci-e2e-test
    resourceVersion: "26897"
    uid: a39fc76c-27ea-409f-a321-2971dd128ebc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x
    namespace: kserve-ci-e2e-test
    resourceVersion: "26197"
    uid: a51b8809-5b40-43d1-bce0-1c4a61a5721f
  kind: Event
  lastTimestamp: "2026-07-28T01:59:07Z"
  message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp
    10.133.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T01:59:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:07Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x.18c651884046e701
    namespace: kserve-ci-e2e-test
    resourceVersion: "26907"
    uid: 1caa8cac-4629-4385-9811-a4df05218913
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-135-95.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-135-95.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:36Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-9f304-predictor-7fbbfcf89b
    namespace: kserve-ci-e2e-test
    resourceVersion: "26186"
    uid: 8b5831a7-50bd-45f6-af72-8e4d84effc99
  kind: Event
  lastTimestamp: "2026-07-28T01:57:36Z"
  message: 'Created pod: raw-sklearn-9f304-predictor-7fbbfcf89b-d785x'
  metadata:
    creationTimestamp: "2026-07-28T01:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:36Z"
    name: raw-sklearn-9f304-predictor-7fbbfcf89b.18c65172e4c6990f
    namespace: kserve-ci-e2e-test
    resourceVersion: "26196"
    uid: 72a1aec1-b1bc-4cac-b10c-d0c60a720965
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:36Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-9f304-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26184"
    uid: ce842c13-5743-4c38-9344-06280935ec65
  kind: Event
  lastTimestamp: "2026-07-28T01:57:36Z"
  message: Scaled up replica set raw-sklearn-9f304-predictor-7fbbfcf89b from 0 to
    1
  metadata:
    creationTimestamp: "2026-07-28T01:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:36Z"
    name: raw-sklearn-9f304-predictor.18c65172e1af9ea3
    namespace: kserve-ci-e2e-test
    resourceVersion: "26187"
    uid: 5db5d1c4-7714-4da9-bce2-98f0db59f892
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:51Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-9f304-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26185"
    uid: 6364afa7-49a5-4d44-af2b-1281929d28cc
  kind: Event
  lastTimestamp: "2026-07-28T01:57:51Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:57:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:51Z"
    name: raw-sklearn-9f304-predictor.18c65176608a7d91
    namespace: kserve-ci-e2e-test
    resourceVersion: "26339"
    uid: 59ea49ff-f9bf-4686-a568-6e6e34e5dd5a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:51Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-9f304-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26185"
    uid: 6364afa7-49a5-4d44-af2b-1281929d28cc
  kind: Event
  lastTimestamp: "2026-07-28T01:57:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:57:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:57:51Z"
    name: raw-sklearn-9f304-predictor.18c6517660eaedc1
    namespace: kserve-ci-e2e-test
    resourceVersion: "26341"
    uid: aaa8e608-6ff2-4e7a-b761-3d59c3af85e6
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:58:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-9f304-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26340"
    uid: 6364afa7-49a5-4d44-af2b-1281929d28cc
  kind: Event
  lastTimestamp: "2026-07-28T01:59:06Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:58:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:59:06Z"
    name: raw-sklearn-9f304-predictor.18c65179dfa297ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "26879"
    uid: 6f8ba357-1e5b-4294-b849-f6072d7a9798
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:58:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-9f304-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26340"
    uid: 6364afa7-49a5-4d44-af2b-1281929d28cc
  kind: Event
  lastTimestamp: "2026-07-28T01:59:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:58:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:59:06Z"
    name: raw-sklearn-9f304-predictor.18c65179e008ce61
    namespace: kserve-ci-e2e-test
    resourceVersion: "26880"
    uid: 69eeb8de-b216-4913-8fdd-601d12d6afda
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-9f304
    namespace: kserve-ci-e2e-test
    resourceVersion: "26176"
    uid: 9a524097-ccc3-4186-a030-58537e73c7be
  kind: Event
  lastTimestamp: "2026-07-28T01:58:58Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:58:58Z"
    name: raw-sklearn-9f304.18c65172e1c92fb5
    namespace: kserve-ci-e2e-test
    resourceVersion: "26819"
    uid: 37007662-b04e-4a71-afea-0f9bebba7546
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:57:36Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-9f304
    namespace: kserve-ci-e2e-test
    resourceVersion: "26176"
    uid: 9a524097-ccc3-4186-a030-58537e73c7be
  kind: Event
  lastTimestamp: "2026-07-28T01:57:36Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-9f304": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-9f304":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:57:36Z"
    name: raw-sklearn-9f304.18c65172e233aed7
    namespace: kserve-ci-e2e-test
    resourceVersion: "26191"
    uid: 4a515b31-c53a-490f-96a8-0563dddb76fe
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:58:58Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-9f304
    namespace: kserve-ci-e2e-test
    resourceVersion: "26817"
    uid: 9a524097-ccc3-4186-a030-58537e73c7be
  kind: Event
  lastTimestamp: "2026-07-28T01:58:58Z"
  message: InferenceService [raw-sklearn-9f304] is Ready
  metadata:
    creationTimestamp: "2026-07-28T01:58:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:58:58Z"
    name: raw-sklearn-9f304.18c65185f1db0fff
    namespace: kserve-ci-e2e-test
    resourceVersion: "26818"
    uid: b0743221-0c20-443e-bfad-04c1768f64c5
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-07-28T01:59:11.558724Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26960"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    to ip-10-0-128-168.ec2.internal
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-07-28T01:59:11Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518918482732
    namespace: kserve-ci-e2e-test
    resourceVersion: "26965"
    uid: a0c52769-833d-4790-9347-d49eed144e26
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-588cf7567-rhr4j
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:11Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-e7c05-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:11Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518924184bdd
    namespace: kserve-ci-e2e-test
    resourceVersion: "26977"
    uid: 2dd87f20-1de7-4c0f-aed2-085e4b58dd74
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26969"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:12Z"
  message: Add eth0 [10.134.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-07-28T01:59:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-07-28T01:59:12Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c651895561cfd0
    namespace: kserve-ci-e2e-test
    resourceVersion: "26987"
    uid: 90d86233-a6dc-4bde-8030-62c39e7005cb
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:12Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:59:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:12Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518956e2170d
    namespace: kserve-ci-e2e-test
    resourceVersion: "26989"
    uid: b9e43910-7a1e-41b4-88c0-7e33afc7b171
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:12Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-07-28T01:59:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:12Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518959b1025b
    namespace: kserve-ci-e2e-test
    resourceVersion: "26990"
    uid: 6acd5f47-50be-4754-9dc5-39827f089475
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:12Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-07-28T01:59:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:12Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c651895a4b5241
    namespace: kserve-ci-e2e-test
    resourceVersion: "26991"
    uid: dce0f787-8020-42ac-a045-4e1f1593ad63
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:16Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:59:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:16Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a3ad8f6f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "27026"
    uid: ed6b280e-43a5-4c4a-b77c-ff1374347b6e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:16Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-07-28T01:59:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:16Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a3de10a3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "27028"
    uid: a5a11d92-bb02-4b4f-b751-8ab4070354a5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:16Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T01:59:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:16Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a3e5c7099
    namespace: kserve-ci-e2e-test
    resourceVersion: "27030"
    uid: bbed753e-6ba9-45f9-ac02-13821227cc03
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:16Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-07-28T01:59:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:16Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a3e682b7c
    namespace: kserve-ci-e2e-test
    resourceVersion: "27031"
    uid: df0c370e-3002-4964-9105-8c51491be5ba
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:16Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-07-28T01:59:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:16Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a4150c3c9
    namespace: kserve-ci-e2e-test
    resourceVersion: "27034"
    uid: 5b7461e1-930b-4f78-8324-822db3858009
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T01:59:16Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T01:59:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T01:59:16Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a41c88fd1
    namespace: kserve-ci-e2e-test
    resourceVersion: "27035"
    uid: 37812472-9252-43c6-be1e-1334ff11d19f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T02:00:43Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-07-28T01:59:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T02:00:43Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6518a76da341b
    namespace: kserve-ci-e2e-test
    resourceVersion: "27674"
    uid: e224ed96-744d-4c2c-9917-3fcbb9641f32
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T02:00:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T02:00:41Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-07-28T02:00:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T02:00:41Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6519e123fc446
    namespace: kserve-ci-e2e-test
    resourceVersion: "27658"
    uid: a03f505b-4525-40a8-9cc2-4f1b8851dd23
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T02:00:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T02:00:41Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-07-28T02:00:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T02:00:41Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6519e1240656a
    namespace: kserve-ci-e2e-test
    resourceVersion: "27659"
    uid: 76501a76-a418-4f75-851e-3812593b7ef4
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T02:00:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz
    namespace: kserve-ci-e2e-test
    resourceVersion: "26964"
    uid: f98494de-7b4b-4603-81f7-032b92198678
  kind: Event
  lastTimestamp: "2026-07-28T02:00:43Z"
  message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp
    10.134.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-07-28T02:00:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-07-28T02:00:43Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz.18c6519e7cf784f4
    namespace: kserve-ci-e2e-test
    resourceVersion: "27673"
    uid: 7fcd5dea-67a0-472b-ac99-04e089efc0a2
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-128-168.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-128-168.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688
    namespace: kserve-ci-e2e-test
    resourceVersion: "26949"
    uid: 0e5df979-1350-480a-ae60-0745c480dad9
  kind: Event
  lastTimestamp: "2026-07-28T01:59:11Z"
  message: 'Created pod: raw-sklearn-runtime-e7c05-predictor-c7b4b4688-j6hlz'
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:59:11Z"
    name: raw-sklearn-runtime-e7c05-predictor-c7b4b4688.18c6518917c21c96
    namespace: kserve-ci-e2e-test
    resourceVersion: "26962"
    uid: cd6cf182-5691-41d4-bf7b-e02ce2952c2e
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-runtime-e7c05-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26947"
    uid: ee50ae4d-3a9f-4dac-ac73-595f7b6ef55b
  kind: Event
  lastTimestamp: "2026-07-28T01:59:11Z"
  message: Scaled up replica set raw-sklearn-runtime-e7c05-predictor-c7b4b4688 from
    0 to 1
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:59:11Z"
    name: raw-sklearn-runtime-e7c05-predictor.18c65189126a5992
    namespace: kserve-ci-e2e-test
    resourceVersion: "26951"
    uid: 0d5206be-918a-49af-993b-0c07c7b5c85c
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-e7c05-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26948"
    uid: 8446abf5-db8a-458d-84ed-87b3db88ac47
  kind: Event
  lastTimestamp: "2026-07-28T01:59:26Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:59:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:59:26Z"
    name: raw-sklearn-runtime-e7c05-predictor.18c6518c91605766
    namespace: kserve-ci-e2e-test
    resourceVersion: "27109"
    uid: 71abfd3a-65ee-4b6b-a838-9e327649cd75
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-e7c05-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "26948"
    uid: 8446abf5-db8a-458d-84ed-87b3db88ac47
  kind: Event
  lastTimestamp: "2026-07-28T01:59:26Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-07-28T01:59:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T01:59:26Z"
    name: raw-sklearn-runtime-e7c05-predictor.18c6518c91d3d45a
    namespace: kserve-ci-e2e-test
    resourceVersion: "27111"
    uid: 28608482-016d-46ce-8717-33d5b013c48f
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:41Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-e7c05-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "27110"
    uid: 8446abf5-db8a-458d-84ed-87b3db88ac47
  kind: Event
  lastTimestamp: "2026-07-28T02:00:41Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:59:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T02:00:41Z"
    name: raw-sklearn-runtime-e7c05-predictor.18c651901089ff8c
    namespace: kserve-ci-e2e-test
    resourceVersion: "27641"
    uid: 89a1a2da-ba3a-464c-86ae-337f899f7764
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:41Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-e7c05-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "27110"
    uid: 8446abf5-db8a-458d-84ed-87b3db88ac47
  kind: Event
  lastTimestamp: "2026-07-28T02:00:41Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-07-28T01:59:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-07-28T02:00:41Z"
    name: raw-sklearn-runtime-e7c05-predictor.18c6519010ecaebf
    namespace: kserve-ci-e2e-test
    resourceVersion: "27642"
    uid: e0b59e5e-3a52-47a3-8654-9083f8741890
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-e7c05
    namespace: kserve-ci-e2e-test
    resourceVersion: "26940"
    uid: f722c7a1-4643-4565-a1d8-fa671d42a8ca
  kind: Event
  lastTimestamp: "2026-07-28T02:00:33Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T02:00:33Z"
    name: raw-sklearn-runtime-e7c05.18c651891277e39f
    namespace: kserve-ci-e2e-test
    resourceVersion: "27586"
    uid: 6d1c0daa-b9f2-4e7b-a6fe-616056e6d412
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-e7c05
    namespace: kserve-ci-e2e-test
    resourceVersion: "26940"
    uid: f722c7a1-4643-4565-a1d8-fa671d42a8ca
  kind: Event
  lastTimestamp: "2026-07-28T01:59:11Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-runtime-e7c05":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-e7c05":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:59:11Z"
    name: raw-sklearn-runtime-e7c05.18c651891311ec43
    namespace: kserve-ci-e2e-test
    resourceVersion: "26971"
    uid: a905add9-4ecf-4ad9-96a0-29b88cb17c08
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T01:59:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-e7c05
    namespace: kserve-ci-e2e-test
    resourceVersion: "26942"
    uid: f722c7a1-4643-4565-a1d8-fa671d42a8ca
  kind: Event
  lastTimestamp: "2026-07-28T01:59:11Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "raw-sklearn-runtime-e7c05": the object
    has been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-07-28T01:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T01:59:11Z"
    name: raw-sklearn-runtime-e7c05.18c651891aee3432
    namespace: kserve-ci-e2e-test
    resourceVersion: "26972"
    uid: 6ed21fff-8114-4272-8669-a0e1b9af52c9
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-07-28T02:00:33Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-e7c05
    namespace: kserve-ci-e2e-test
    resourceVersion: "27584"
    uid: f722c7a1-4643-4565-a1d8-fa671d42a8ca
  kind: Event
  lastTimestamp: "2026-07-28T02:00:33Z"
  message: InferenceService [raw-sklearn-runtime-e7c05] is Ready
  metadata:
    creationTimestamp: "2026-07-28T02:00:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-07-28T02:00:33Z"
    name: raw-sklearn-runtime-e7c05.18c6519c2f1fcf3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "27585"
    uid: ac2b4e9e-01bb-4f65-ac4e-d8a1575a9478
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
kind: EventList
metadata:
  resourceVersion: "27930"