---
apiVersion: v1
items:
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:00:12.394020Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21367"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-68984b9498-l98dp
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b433605787
    namespace: kserve-ci-e2e-test
    resourceVersion: "21371"
    uid: f4c21589-1596-4840-96a4-d893213447f1
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-create-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b44135598f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21386"
    uid: 4aad10ef-f194-4300-b7a2-3abda66d2eb3
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21376"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Add eth0 [10.134.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b47919b099
    namespace: kserve-ci-e2e-test
    resourceVersion: "21406"
    uid: 2098b490-16ce-4ba7-8c5e-7a3c6203f94d
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b47a59d5f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21410"
    uid: 5235eedd-3e47-474f-b803-daf11732214f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b47deb740c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21413"
    uid: 0e1b6e46-849e-4780-b3b7-290f9ed59186
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b47e5fe468
    namespace: kserve-ci-e2e-test
    resourceVersion: "21414"
    uid: fdcbc8fe-3b01-4cdf-b0df-feb58cfa11a9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b5981a6e1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21475"
    uid: 060bec40-3f74-4792-866d-9c3c6c2dfbe5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b59b91d3a1
    namespace: kserve-ci-e2e-test
    resourceVersion: "21480"
    uid: f9d7df4a-ad1a-4852-bd8a-b7e5aa5afcf4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b59c0901fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "21484"
    uid: 54c4538b-da2c-4c8c-9768-eaa49166fb09
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b59c143445
    namespace: kserve-ci-e2e-test
    resourceVersion: "21485"
    uid: 9f651212-6786-45ce-a0b1-a79aad1712eb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b59fb03fa5
    namespace: kserve-ci-e2e-test
    resourceVersion: "21487"
    uid: 61bc9556-3aea-49b7-a092-ab65fe34507b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b5a024cb44
    namespace: kserve-ci-e2e-test
    resourceVersion: "21489"
    uid: 702eb4bf-94f4-4531-a6f1-4aba670465da
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:01:45Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:00:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:45Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682b5d4357d9e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22193"
    uid: 82ed6668-7e5b-4b6d-a0a5-be1be02d5803
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:01:42Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:01:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:42Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682c927772b32
    namespace: kserve-ci-e2e-test
    resourceVersion: "22170"
    uid: 9317256c-77fa-4687-a2d2-cc9754812366
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:01:42Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:01:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:42Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682c927784530
    namespace: kserve-ci-e2e-test
    resourceVersion: "22171"
    uid: 21dfd6d4-cd23-4b31-805f-84a7c551e377
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-l98dp
    namespace: kserve-ci-e2e-test
    resourceVersion: "21369"
    uid: 16afa416-1cf6-45d2-b374-54515798f714
  kind: Event
  lastTimestamp: "2026-09-18T20:01:45Z"
  message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp
    10.134.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:01:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:45Z"
    name: isvc-canary-create-predictor-68984b9498-l98dp.18d682c9da727d10
    namespace: kserve-ci-e2e-test
    resourceVersion: "22191"
    uid: 1837abf1-ef49-4532-9081-44c3024cc50e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:00:12.351640Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21358"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-68984b9498-rd2x9
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b430d9bb3e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21362"
    uid: a1034675-08a6-4d62-831c-8900bca69d77
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-create-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b43cbbbcd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "21383"
    uid: 9162486d-3de8-4155-9e54-b4276aa27cc6
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21366"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Add eth0 [10.133.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b476b83dee
    namespace: kserve-ci-e2e-test
    resourceVersion: "21403"
    uid: 83d8819b-6dcc-4e0b-9048-ed24af9f3ba8
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b477f53a30
    namespace: kserve-ci-e2e-test
    resourceVersion: "21405"
    uid: 98a8ea42-811e-4beb-ba8c-35af947ce50f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b47b883ee5
    namespace: kserve-ci-e2e-test
    resourceVersion: "21411"
    uid: 11757e1f-0ac0-469d-a231-fc4f9ec3dca7
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b47c021344
    namespace: kserve-ci-e2e-test
    resourceVersion: "21412"
    uid: 7f332750-414f-4ffe-ad3a-3882a23f6437
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b58609185a
    namespace: kserve-ci-e2e-test
    resourceVersion: "21464"
    uid: fe2aeaa0-e3db-48f5-8bb5-f94e430f69dc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b589368a64
    namespace: kserve-ci-e2e-test
    resourceVersion: "21466"
    uid: cc695e41-d3d3-4b8a-a035-4cb596d7e72f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b589a37268
    namespace: kserve-ci-e2e-test
    resourceVersion: "21467"
    uid: 05f0da77-404c-47b6-ad72-68ecb82b6cf8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b589aca8f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21469"
    uid: dee9f4d0-6b16-47c3-8050-b46e1644299a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b58c6a80d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21471"
    uid: b54bfbe8-0103-4690-a732-6dc3e4974778
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b58cd88237
    namespace: kserve-ci-e2e-test
    resourceVersion: "21472"
    uid: 0edec2a1-3520-49f0-8151-448c5d261670
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:01:26Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:00:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:26Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682b5c212ee90
    namespace: kserve-ci-e2e-test
    resourceVersion: "21987"
    uid: a498c9ed-e2a9-4f80-926e-594a79312c86
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682c794539a9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22074"
    uid: 11d7e11e-90b0-49a0-a827-78355eba5c68
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682c79454276c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22078"
    uid: 5b1af025-d795-4127-a3c2-9ca64e076aa7
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-68984b9498-rd2x9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21360"
    uid: d50bd5e5-8231-46d5-b611-a10f73987fca
  kind: Event
  lastTimestamp: "2026-09-18T20:01:36Z"
  message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp
    10.133.0.33:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:01:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:36Z"
    name: isvc-canary-create-predictor-68984b9498-rd2x9.18d682c7b0143733
    namespace: kserve-ci-e2e-test
    resourceVersion: "22097"
    uid: afe20303-37b2-4d57-92c5-84a4444dfb7b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-68984b9498
    namespace: kserve-ci-e2e-test
    resourceVersion: "21343"
    uid: ecb928ed-9d74-4049-95ce-bf705e5f7c68
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'Created pod: isvc-canary-create-predictor-68984b9498-rd2x9'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor-68984b9498.18d682b430517819
    namespace: kserve-ci-e2e-test
    resourceVersion: "21359"
    uid: c7c8bba5-c019-4e32-8c5d-8b0905013105
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-68984b9498
    namespace: kserve-ci-e2e-test
    resourceVersion: "21343"
    uid: ecb928ed-9d74-4049-95ce-bf705e5f7c68
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'Created pod: isvc-canary-create-predictor-68984b9498-l98dp'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor-68984b9498.18d682b432f9c11e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21368"
    uid: 82330a4b-2a3d-43d7-8e2f-3452a5280199
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:35Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-68984b9498
    namespace: kserve-ci-e2e-test
    resourceVersion: "22069"
    uid: ecb928ed-9d74-4049-95ce-bf705e5f7c68
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: 'Deleted pod: isvc-canary-create-predictor-68984b9498-rd2x9'
  metadata:
    creationTimestamp: "2026-09-18T20:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create-predictor-68984b9498.18d682c7943dbf85
    namespace: kserve-ci-e2e-test
    resourceVersion: "22076"
    uid: 022eefc7-638a-42f5-b5fc-8d9dd23e5a80
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21342"
    uid: 1e1138b7-6924-4ae2-a4b9-2fafc382745c
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: Scaled up replica set isvc-canary-create-predictor-68984b9498 from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-predictor.18d682b42d3ebad6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21345"
    uid: 3f432877-dc89-4202-ac28-6147fce90c0f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:35Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22064"
    uid: 1e1138b7-6924-4ae2-a4b9-2fafc382745c
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: Scaled down replica set isvc-canary-create-predictor-68984b9498 from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-18T20:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create-predictor.18d682c7930c30d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22070"
    uid: 4eb3eed7-c38a-464f-b022-aa94c1c040cf
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:00:12.316999Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21349"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b42ec92d19
    namespace: kserve-ci-e2e-test
    resourceVersion: "21355"
    uid: 071bb6ca-0f66-42a2-a0ec-4b51799bc9a7
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-create-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b43b30fa28
    namespace: kserve-ci-e2e-test
    resourceVersion: "21382"
    uid: 37458411-5c91-4184-8bf2-56c0d086f420
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21361"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Add eth0 [10.134.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b474a33cfb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21400"
    uid: 541656c1-acbe-4442-8c6e-ecf8b3faa4b6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b475e969d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21402"
    uid: b3775dbf-3289-4e3b-9bcd-712cd57bca6f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b4797129c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21407"
    uid: 7324e64f-061a-48d1-9c48-f88706f8472f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:13Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:00:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:13Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b479e63faa
    namespace: kserve-ci-e2e-test
    resourceVersion: "21409"
    uid: 42631d05-497a-4510-9685-b8392a602027
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b598054fad
    namespace: kserve-ci-e2e-test
    resourceVersion: "21474"
    uid: e265e95c-557b-459f-972a-87d3319fc793
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b59b87de04
    namespace: kserve-ci-e2e-test
    resourceVersion: "21478"
    uid: 498f8a81-4c89-455d-9040-b46e88ca65ed
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b59bfbf078
    namespace: kserve-ci-e2e-test
    resourceVersion: "21482"
    uid: efbc4f71-a1f4-47f8-be30-537d3b721608
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b59c06b132
    namespace: kserve-ci-e2e-test
    resourceVersion: "21483"
    uid: 83177dc3-c189-4d0c-9633-0e5bfd7cff6d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b59fbada88
    namespace: kserve-ci-e2e-test
    resourceVersion: "21488"
    uid: 4382fa3d-c0e9-42fb-b84c-c740e8634079
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:00:18Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:00:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:18Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b5a0310df8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21490"
    uid: fa431d4a-f3fb-489c-b684-a8117fd5be78
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:45Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:00:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:45Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682b5d41ddc19
    namespace: kserve-ci-e2e-test
    resourceVersion: "22194"
    uid: 8b332895-fea7-4f7a-8e22-043c445307e4
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:42Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:01:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:42Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682c92726dc30
    namespace: kserve-ci-e2e-test
    resourceVersion: "22167"
    uid: fa2a0b02-8c93-4c38-b3b3-c18d83d131a2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:42Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:01:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:42Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682c927278585
    namespace: kserve-ci-e2e-test
    resourceVersion: "22168"
    uid: 4208069c-9a68-4d1d-8603-01028685bcd9
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21352"
    uid: 261da5a3-9952-47a6-a373-3bdb58c7c8e1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:45Z"
  message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp
    10.134.0.31:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:01:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:45Z"
    name: isvc-canary-create-v2-predictor-579784d7d7-wfkvd.18d682c9da8fd589
    namespace: kserve-ci-e2e-test
    resourceVersion: "22192"
    uid: 0931386a-9d47-4ce3-8e74-426d011a8cec
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-v2-predictor-579784d7d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21334"
    uid: c591988e-cadb-4900-9da3-e4c0b0d58e20
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'Created pod: isvc-canary-create-v2-predictor-579784d7d7-wfkvd'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-v2-predictor-579784d7d7.18d682b42e55ba3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21351"
    uid: c969c449-b4ab-4b5f-8c4c-91033898a5af
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21333"
    uid: b4d7afbc-6647-4be2-8543-980293c97999
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: Scaled up replica set isvc-canary-create-v2-predictor-579784d7d7 from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-canary-create-v2-predictor.18d682b42b6227bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "21335"
    uid: ae4a973a-620c-4771-bdcc-ea2e06b4a666
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 21
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21325"
    uid: 8f002fac-641d-40f6-985d-d6aa3123afd1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create.18d682b42cfbb765
    namespace: kserve-ci-e2e-test
    resourceVersion: "22094"
    uid: 80b10519-d96b-4527-8e6d-965e93757a2d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21325"
    uid: 8f002fac-641d-40f6-985d-d6aa3123afd1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: 'Failed to update status for InferenceService "isvc-canary-create": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create.18d682b42d781a13
    namespace: kserve-ci-e2e-test
    resourceVersion: "22090"
    uid: c9920eee-546f-46a1-946d-402a058bd7e9
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:12Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21325"
    uid: 8f002fac-641d-40f6-985d-d6aa3123afd1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T20:00:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create.18d682b42d7878a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22091"
    uid: 304319f2-b8b8-4a79-bede-602e323aa2be
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:35Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "22085"
    uid: 8f002fac-641d-40f6-985d-d6aa3123afd1
  kind: Event
  lastTimestamp: "2026-09-18T20:01:35Z"
  message: InferenceService [isvc-canary-create] is Ready
  metadata:
    creationTimestamp: "2026-09-18T20:01:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:01:35Z"
    name: isvc-canary-create.18d682c79b0b5b0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22086"
    uid: 216a0896-1101-42c0-8ecc-bbb7ce6b4712
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:01:47.574888Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22251"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-cb75746f8-grg87
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682ca5c9928af
    namespace: kserve-ci-e2e-test
    resourceVersion: "22257"
    uid: 4efd806f-6817-4a79-9b26-b14f86c2c9fb
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22264"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Add eth0 [10.134.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682ca7e9e74d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "22293"
    uid: c4e8e6c3-1621-4a77-866e-1e77c0913ec4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682ca7ff126f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22299"
    uid: 660211e7-58fc-412b-88c4-ac491e623569
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682ca8299508f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22301"
    uid: 5a8557cb-c4a0-4505-9dfc-54540f00a551
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682ca83082b89
    namespace: kserve-ci-e2e-test
    resourceVersion: "22302"
    uid: 41e39f82-b13d-4f52-97a7-ec16633f9d40
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cb910e0996
    namespace: kserve-ci-e2e-test
    resourceVersion: "22348"
    uid: 0338ca2b-ef77-41f6-af1f-861c3a7b0df4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cb94d42280
    namespace: kserve-ci-e2e-test
    resourceVersion: "22352"
    uid: 28623630-050b-461e-9a51-49a2ac0afc5c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cb955dc219
    namespace: kserve-ci-e2e-test
    resourceVersion: "22358"
    uid: 5854d2bc-7e38-493c-98e5-209ce17c14de
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cb95682163
    namespace: kserve-ci-e2e-test
    resourceVersion: "22359"
    uid: 924ff32c-43ad-4c95-a969-b21df1549e3f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cb988749cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22361"
    uid: 39e59046-d062-4c8d-b6f5-6f048d38b061
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cb98f8bbc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "22364"
    uid: 15136301-3669-4e0a-af00-54c71c7da1f3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:03:00Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:01:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:00Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682cc092e9290
    namespace: kserve-ci-e2e-test
    resourceVersion: "22871"
    uid: d540f41c-9023-42e2-aecd-1fa62a78a643
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:03:12Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:12Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682de07e38749
    namespace: kserve-ci-e2e-test
    resourceVersion: "22982"
    uid: 05c3340a-5a09-4bdd-bec4-8df3f026b0cb
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:03:12Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:03:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:12Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682de07e3dea5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22983"
    uid: e1e62705-a3e2-4ca3-807e-095ebf18fb0f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-grg87
    namespace: kserve-ci-e2e-test
    resourceVersion: "22254"
    uid: 0c248f96-4bea-454e-b5c9-8c144cfa7ee4
  kind: Event
  lastTimestamp: "2026-09-18T20:03:15Z"
  message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp
    10.134.0.34:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:03:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:15Z"
    name: isvc-canary-promote-predictor-cb75746f8-grg87.18d682dee55dde31
    namespace: kserve-ci-e2e-test
    resourceVersion: "23012"
    uid: 1380ee2d-03d7-4f4c-b59c-4874f592311c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:01:47.624016Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22265"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-cb75746f8-zgvp4
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682ca5f86d240
    namespace: kserve-ci-e2e-test
    resourceVersion: "22269"
    uid: f75f4828-99d7-4c6d-8cec-9e4359bdf9b6
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:47Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682ca68ea1778
    namespace: kserve-ci-e2e-test
    resourceVersion: "22284"
    uid: 32e27d22-e0bf-497c-865b-3db9dea967f4
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22273"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Add eth0 [10.133.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682caa5688b9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22312"
    uid: 11eb871c-0272-423a-a31a-47916fca6fff
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682caa69efa3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "22315"
    uid: c69dbb74-0358-4c81-a9b7-f4fb8598bede
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682caa992a8cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "22317"
    uid: 3d910555-0ba0-49bc-8060-95e9fbfd8d68
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682caaa071f57
    namespace: kserve-ci-e2e-test
    resourceVersion: "22318"
    uid: a8f98489-f027-467c-9dea-8b2b614ac717
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:53Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:53Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbbd3c6de9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22372"
    uid: c9166650-b77f-47df-883a-ce405e492c24
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:53Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:53Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbc061ed6b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22374"
    uid: 5cad7dbf-f255-43a8-b95d-55df1c88d3b8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:53Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:53Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbc0d9a898
    namespace: kserve-ci-e2e-test
    resourceVersion: "22375"
    uid: e924250b-f47d-437c-8bd7-b5f45ec2093a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:53Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:53Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbc0e3a39c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22377"
    uid: 182e58d6-625b-4a97-b47b-3825b62fc05d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:53Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:53Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbc3ab7e95
    namespace: kserve-ci-e2e-test
    resourceVersion: "22378"
    uid: 1ffa31a2-a73f-4fb0-831c-68b1f869b6d5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:01:53Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:53Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbc4303fb7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22379"
    uid: 3fd0e8f6-0a4a-443e-b71e-da5100e8432e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:03:20Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:01:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:20Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682cbf94a7b3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23089"
    uid: a7f2b560-4fe5-4415-b8a8-06e64f6772ca
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:03:17Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:17Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682df5aecef41
    namespace: kserve-ci-e2e-test
    resourceVersion: "23039"
    uid: 12981026-b21d-4a51-85e9-755a0eebdcf3
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:03:17Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:17Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682df5aed69a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "23040"
    uid: 66c91230-387f-4874-99fa-e5f245b90e9d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22267"
    uid: 45cdd521-d5fa-431d-9d4e-20c68e5cfd9a
  kind: Event
  lastTimestamp: "2026-09-18T20:03:20Z"
  message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp
    10.133.0.34:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:03:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:20Z"
    name: isvc-canary-promote-predictor-cb75746f8-zgvp4.18d682dfff87320d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23088"
    uid: 7d4f5d55-c808-4143-ba79-21cc97a5e8f2
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-cb75746f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "22240"
    uid: 19df5365-0a0a-4708-8dbe-981af3ca01d3
  kind: Event
  lastTimestamp: "2026-09-18T20:01:47Z"
  message: 'Created pod: isvc-canary-promote-predictor-cb75746f8-grg87'
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-predictor-cb75746f8.18d682ca5c37eadc
    namespace: kserve-ci-e2e-test
    resourceVersion: "22253"
    uid: dea190f8-6f77-41ff-b306-b772e346666b
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-cb75746f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "22240"
    uid: 19df5365-0a0a-4708-8dbe-981af3ca01d3
  kind: Event
  lastTimestamp: "2026-09-18T20:01:47Z"
  message: 'Created pod: isvc-canary-promote-predictor-cb75746f8-zgvp4'
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-predictor-cb75746f8.18d682ca5f1af09e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22266"
    uid: 4ee2e05c-0112-4c63-a913-1570d5a5ac2d
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-cb75746f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "22967"
    uid: 19df5365-0a0a-4708-8dbe-981af3ca01d3
  kind: Event
  lastTimestamp: "2026-09-18T20:03:11Z"
  message: 'Deleted pod: isvc-canary-promote-predictor-cb75746f8-grg87'
  metadata:
    creationTimestamp: "2026-09-18T20:03:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:03:11Z"
    name: isvc-canary-promote-predictor-cb75746f8.18d682ddebcc180d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22970"
    uid: 96c9a1a9-9d98-4638-8e3a-3621a7344819
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22239"
    uid: 56b2588e-cd7b-4d28-8a40-f86792cd3f61
  kind: Event
  lastTimestamp: "2026-09-18T20:01:47Z"
  message: Scaled up replica set isvc-canary-promote-predictor-cb75746f8 from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-predictor.18d682ca5901f1f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "22242"
    uid: 5b1fddac-b10d-4d6e-ac4b-26a621d7a481
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22963"
    uid: 56b2588e-cd7b-4d28-8a40-f86792cd3f61
  kind: Event
  lastTimestamp: "2026-09-18T20:03:11Z"
  message: Scaled down replica set isvc-canary-promote-predictor-cb75746f8 from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-18T20:03:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:03:11Z"
    name: isvc-canary-promote-predictor.18d682ddeb6311a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22968"
    uid: 723aea7a-a2aa-4149-b1e7-11c88ceba4b0
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:03:17.858607Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23049"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:03:17Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:03:17Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682df61ed5fb1
    namespace: kserve-ci-e2e-test
    resourceVersion: "23054"
    uid: 71d893c6-608c-4ed6-8a9f-52385214ee09
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:18Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23059"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:18Z"
  message: Add eth0 [10.133.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:03:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:03:18Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682df83d473fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "23063"
    uid: 7d72c9e9-11fa-415c-b379-182471f41394
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:18Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:03:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:18Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682df8549feb1
    namespace: kserve-ci-e2e-test
    resourceVersion: "23065"
    uid: c4c28c3d-f6c9-4571-8a9c-505c43963f0a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:18Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:03:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:18Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682df8832a1a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23066"
    uid: 0494829f-3ceb-4145-a109-5ef3367e6f1b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:18Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:18Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:03:18Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:18Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682df88a2b5e2
    namespace: kserve-ci-e2e-test
    resourceVersion: "23067"
    uid: 1a44aa2e-a9cc-49d0-b489-b67b5a6b0008
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:22Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:03:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:22Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e08dd8a64e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23106"
    uid: 3c4ae7e9-5092-4825-a821-06c58656ab8b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:22Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:03:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:22Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e091356c52
    namespace: kserve-ci-e2e-test
    resourceVersion: "23111"
    uid: 16273cc9-2e83-4fb3-95f1-2b35def4228f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:22Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:03:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:22Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e091b6bf4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23112"
    uid: 0bd4849d-afa0-4958-9a79-c960de87285d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:22Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:03:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:22Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e091c033d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "23114"
    uid: 6a89df69-a7cc-4381-b7c6-91c2f7947ba4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:22Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:03:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:23Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e0945d0be2
    namespace: kserve-ci-e2e-test
    resourceVersion: "23115"
    uid: e215aeae-9327-4ea1-9eb6-75da013ca1dc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:03:23Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:03:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:03:23Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e094cac004
    namespace: kserve-ci-e2e-test
    resourceVersion: "23116"
    uid: afe1d059-91a6-4c2f-b22d-0c935b6d7034
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:04:33Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:03:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:33Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682e0ca14de80
    namespace: kserve-ci-e2e-test
    resourceVersion: "23579"
    uid: 81a9a43e-fa71-433b-9228-1d78320017d0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:04:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:04:47Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:04:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:47Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682f45c56439d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23699"
    uid: f47c6eca-011d-4e12-8e19-9981d310e521
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:04:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:04:47Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:04:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:47Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682f45c56b7f5
    namespace: kserve-ci-e2e-test
    resourceVersion: "23701"
    uid: 08475c71-07b9-4e51-bc5e-4ec65e0624ca
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:04:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "23051"
    uid: 21853fe9-4cf1-40b4-8b7a-6a80ac0d9c74
  kind: Event
  lastTimestamp: "2026-09-18T20:04:49Z"
  message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp
    10.133.0.35:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:04:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:49Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl.18d682f4d03c0c86
    namespace: kserve-ci-e2e-test
    resourceVersion: "23716"
    uid: 3b2d4fb8-7091-496c-9045-d7ae7eea3526
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:01:47.564598Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22247"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682ca5bfc2ef7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22252"
    uid: 81f8cdc7-a067-47d3-b51f-23749ea93e31
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22260"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Add eth0 [10.134.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682ca7e2b2e07
    namespace: kserve-ci-e2e-test
    resourceVersion: "22292"
    uid: 9f7c1614-2397-445d-be36-fbadf987d84c
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682ca7f6ae488
    namespace: kserve-ci-e2e-test
    resourceVersion: "22296"
    uid: cdc8972d-b73f-41dd-b282-c00ad600aacd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682ca828e6455
    namespace: kserve-ci-e2e-test
    resourceVersion: "22300"
    uid: d15c811f-5fa4-401a-ba4e-9e8c93d4795b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:48Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:01:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:48Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682ca8315a18e
    namespace: kserve-ci-e2e-test
    resourceVersion: "22303"
    uid: 9d0c6e24-8a51-4a75-84da-755592027acb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cb912d4aff
    namespace: kserve-ci-e2e-test
    resourceVersion: "22349"
    uid: d14ba9e3-e32e-4500-9f58-ed9bad3dc9bb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cb94de3791
    namespace: kserve-ci-e2e-test
    resourceVersion: "22353"
    uid: c840d113-782e-4bd5-acc1-758f0044a43d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cb95516f9b
    namespace: kserve-ci-e2e-test
    resourceVersion: "22355"
    uid: 09a1e5de-91f9-46b2-907f-2b0c3fe9d856
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cb955c94ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "22357"
    uid: 3135cc87-7660-43e8-8b15-c582072642ee
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cb9895c48f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22363"
    uid: 130fe793-d2d4-4006-b3e9-302a36a495e7
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:01:52Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:01:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:01:52Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cb990636a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22365"
    uid: b000a863-52ca-4ed0-88cc-69acf883b5d0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:04:51Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:01:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:51Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682cbcd3c94c4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23725"
    uid: caa631a7-13f7-451e-a656-6b65dc8fe8e6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:04:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:04:47Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:04:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:47Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682f45c53416f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23700"
    uid: f5670f8d-5020-44b4-a1b9-a3a5b3ed0479
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:04:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:04:47Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:04:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:47Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682f45c5417fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "23702"
    uid: 72cb3b50-66de-42dd-9cbd-c6f2c35def5c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:04:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "22249"
    uid: 7fa53863-2ece-4079-8590-ba1b442137c2
  kind: Event
  lastTimestamp: "2026-09-18T20:04:49Z"
  message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp
    10.134.0.33:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:04:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:04:49Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p.18d682f4c7eeb33c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23714"
    uid: 4656afd3-95f6-4dfa-9119-b49c8a464428
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-c6f7b798f
    namespace: kserve-ci-e2e-test
    resourceVersion: "22231"
    uid: c875fdd4-12bc-4225-b713-42ba1ee7c3ce
  kind: Event
  lastTimestamp: "2026-09-18T20:01:47Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-c6f7b798f-g9c8p'
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f.18d682ca5b92f422
    namespace: kserve-ci-e2e-test
    resourceVersion: "22248"
    uid: 83876afb-4c88-4736-a9db-913c0de61429
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:17Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-c6f7b798f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23046"
    uid: c875fdd4-12bc-4225-b713-42ba1ee7c3ce
  kind: Event
  lastTimestamp: "2026-09-18T20:03:17Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-c6f7b798f-dllbl'
  metadata:
    creationTimestamp: "2026-09-18T20:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:03:17Z"
    name: isvc-canary-promote-v2-predictor-c6f7b798f.18d682df615fb87a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23050"
    uid: 9fea11af-04e2-4a8c-8912-a711840ab6a8
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22230"
    uid: fc1065a8-f0cb-4f23-8032-43b553deacf3
  kind: Event
  lastTimestamp: "2026-09-18T20:01:47Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-c6f7b798f from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:01:47Z"
    name: isvc-canary-promote-v2-predictor.18d682ca56fe2009
    namespace: kserve-ci-e2e-test
    resourceVersion: "22232"
    uid: 763f974c-12e8-43d7-9383-1ba13a0ae86d
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:17Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23042"
    uid: fc1065a8-f0cb-4f23-8032-43b553deacf3
  kind: Event
  lastTimestamp: "2026-09-18T20:03:17Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-c6f7b798f from 1
    to 2
  metadata:
    creationTimestamp: "2026-09-18T20:03:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:03:17Z"
    name: isvc-canary-promote-v2-predictor.18d682df5d3502f2
    namespace: kserve-ci-e2e-test
    resourceVersion: "23047"
    uid: 9d976430-9c23-42e7-a224-5a1e4a6b8cb7
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22223"
    uid: d24771c3-3fb8-4528-a4d1-d6258317a82f
  kind: Event
  lastTimestamp: "2026-09-18T20:03:10Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:03:10Z"
    name: isvc-canary-promote.18d682ca58b31d72
    namespace: kserve-ci-e2e-test
    resourceVersion: "22952"
    uid: 0814900f-4e93-4373-8a5b-4c9373199983
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22223"
    uid: d24771c3-3fb8-4528-a4d1-d6258317a82f
  kind: Event
  lastTimestamp: "2026-09-18T20:03:10Z"
  message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:03:10Z"
    name: isvc-canary-promote.18d682ca591cee6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22947"
    uid: 83cbadc4-1cc0-4afc-8174-90efdbc2d4fb
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:01:47Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22223"
    uid: d24771c3-3fb8-4528-a4d1-d6258317a82f
  kind: Event
  lastTimestamp: "2026-09-18T20:03:10Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T20:01:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:03:10Z"
    name: isvc-canary-promote.18d682ca591d32f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "22948"
    uid: 5849ff4c-84f5-4965-8fe3-82225b50c58c
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T20:03:10Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22950"
    uid: d24771c3-3fb8-4528-a4d1-d6258317a82f
  kind: Event
  lastTimestamp: "2026-09-18T20:04:43Z"
  message: InferenceService [isvc-canary-promote] is Ready
  metadata:
    creationTimestamp: "2026-09-18T20:03:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:04:43Z"
    name: isvc-canary-promote.18d682ddb4f50455
    namespace: kserve-ci-e2e-test
    resourceVersion: "23655"
    uid: fd727641-2a1c-4dfa-af70-5f6c97696497
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:50:06.100142Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15095"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d68227097200e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15098"
    uid: 2d4b71bb-7af9-444c-ba0b-94da00e0a16a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15105"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: Add eth0 [10.134.0.18/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d682272b8a41dd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15152"
    uid: 5f26ec32-177e-4cee-b367-1b78de195868
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d682272cddb7f5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15154"
    uid: ba000a87-4160-413b-9ad3-a5d4f36be81e
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:10Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    in 3.408s (3.408s including waiting). Image size: 306813343 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:10Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d68227f807439f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15201"
    uid: 041b2a69-df2f-4289-9766-8c9ec313db05
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:10Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:50:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:10Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d68227fb76414c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15204"
    uid: 1641f8ca-11c8-4387-b332-bb08274883cc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:10Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:50:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:10Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d68227fbea7cf5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15206"
    uid: 64200446-32d5-4464-a250-028507e48935
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:13Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
  metadata:
    creationTimestamp: "2026-09-18T19:50:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:13Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d68228db1962a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15250"
    uid: 4c390be7-a958-47dc-9d31-9d91b033ac33
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    in 16.205s (16.205s including waiting). Image size: 1583067017 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822ca1065457
    namespace: kserve-ci-e2e-test
    resourceVersion: "15384"
    uid: 6415b6ab-c406-49af-8981-cd1f44ffd4bd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822ca43e4d2d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15387"
    uid: f7555192-aa6c-44dc-b0c1-80d4e748c4d9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822ca4d43b8d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15391"
    uid: 764c9016-73a9-44c1-acb6-18ca6fb6ef92
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822ca4e1ec24
    namespace: kserve-ci-e2e-test
    resourceVersion: "15392"
    uid: 1ef208ce-81f9-477e-8b8c-ce1a74140366
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:32Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.761s (2.761s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:32Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822d4981c890
    namespace: kserve-ci-e2e-test
    resourceVersion: "15415"
    uid: 34bf5d68-f079-4a18-ad7f-7db804c80183
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822d4e6d8f45
    namespace: kserve-ci-e2e-test
    resourceVersion: "15418"
    uid: 64d5e9f7-513f-4f8f-bdc2-a3af94e42537
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822d4fb997d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "15419"
    uid: 58ea612f-6188-413f-af96-2ff793c75154
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:51:40Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:50:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:40Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6822d8b343aa6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15960"
    uid: e5ec9f73-fb11-456b-b821-794fac713a61
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:51:51Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:51:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:51Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6823f808e45e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16076"
    uid: 627de204-2968-4775-8d3a-fe9cc9cb0aa1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:51:51Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:51:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:51Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6823f808eed45
    namespace: kserve-ci-e2e-test
    resourceVersion: "16077"
    uid: 4ebadbbd-279a-4742-a8fb-fef0a15328a8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk
    namespace: kserve-ci-e2e-test
    resourceVersion: "15097"
    uid: 0b7c9b7d-6459-4084-9ed4-10acd85aa3e2
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp
    10.134.0.18:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:51:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk.18d6824067622f44
    namespace: kserve-ci-e2e-test
    resourceVersion: "16109"
    uid: d12a0f79-1241-457f-a3cb-b38ad07cc15c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:50:06.211786Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15116"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822710197d11
    namespace: kserve-ci-e2e-test
    resourceVersion: "15122"
    uid: 8abd9279-f62f-4b4b-9ae4-ba3a8f9710f7
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: 'MountVolume.SetUp failed for volume "kube-rbac-proxy-sar-config" : failed
    to sync configmap cache: timed out waiting for the condition'
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d682275ad74eed
    namespace: kserve-ci-e2e-test
    resourceVersion: "15165"
    uid: d8a7e501-147f-47d7-87f9-cf7ae8459041
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : failed to sync secret
    cache: timed out waiting for the condition'
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d682275ad88f64
    namespace: kserve-ci-e2e-test
    resourceVersion: "15166"
    uid: 403e7192-45ff-4050-83ab-e211da188384
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:08Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15143"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:08Z"
  message: Add eth0 [10.133.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:50:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:50:08Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d682278bf08658
    namespace: kserve-ci-e2e-test
    resourceVersion: "15180"
    uid: ede37701-12fb-4441-8b5a-934b866157ae
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:08Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:50:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:08Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d682278d2e359b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15183"
    uid: d445bc65-8470-4a07-911b-c2c38a10d60a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:08Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:50:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:08Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d68227905c1d05
    namespace: kserve-ci-e2e-test
    resourceVersion: "15184"
    uid: 17a3858c-de4d-42a4-abf6-0acfba9ec1b4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:08Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:50:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:08Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822790d265a1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15185"
    uid: 7aefabe5-5dbb-4fc4-8c4d-cea8b53115c1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:12Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
  metadata:
    creationTimestamp: "2026-09-18T19:50:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:12Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822875ccafa0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15230"
    uid: 0ca77612-5c39-4f41-bb52-386aabd1f7fa
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    in 15.998s (15.998s including waiting). Image size: 1583067017 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822c2f6975af
    namespace: kserve-ci-e2e-test
    resourceVersion: "15365"
    uid: a8cbe57f-7239-493f-9591-db0a8f55ba75
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822c32da025d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15367"
    uid: 3623a5b7-cc89-4ed7-823f-3e04fe11c4f6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822c334cfb7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15368"
    uid: 63f1cba8-4d8f-499b-96c2-8d211652c5a8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822c3359d8af
    namespace: kserve-ci-e2e-test
    resourceVersion: "15369"
    uid: bf6c9120-3d26-46b9-bc21-6bd39395ccff
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.646s (2.646s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822cd11cb4be
    namespace: kserve-ci-e2e-test
    resourceVersion: "15397"
    uid: 2d9314df-00c6-4dd9-9e92-d544bb9ddff1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:50:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822cd54506e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15401"
    uid: 0c5ae1c9-6dd2-45ab-b5e2-6d22812ee076
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:50:31Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:50:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:31Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822cd5d3fda0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15404"
    uid: 4a10a510-70d6-4b3f-a7ce-46feaf8c0e64
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:51:38Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:50:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:38Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6822d26a68f05
    namespace: kserve-ci-e2e-test
    resourceVersion: "15945"
    uid: 5cbe86ee-438c-4c23-bd37-59305055e720
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:53:26Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:26Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6825598bac0ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "17029"
    uid: d55f0222-72ef-4da8-b360-969a254c0c44
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:53:26Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:26Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6825598bb2a7e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17031"
    uid: 9c0667f9-b3cc-4600-89b0-e8afaf17ae6c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz
    namespace: kserve-ci-e2e-test
    resourceVersion: "15117"
    uid: 7f178a42-ce5a-47cd-a20c-67658eaed665
  kind: Event
  lastTimestamp: "2026-09-18T19:53:28Z"
  message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp
    10.133.0.27:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz.18d6825621537a7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17050"
    uid: 9e6afed3-099d-4b74-a26c-94358039ae57
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:51:56.012263Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16140"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68240a0b7f5a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16146"
    uid: fffe3e67-c346-43c5-a467-333a541b1931
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:56Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16150"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:51:56Z"
  message: Add eth0 [10.134.0.19/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68240c2c8e459
    namespace: kserve-ci-e2e-test
    resourceVersion: "16159"
    uid: 127f3f13-c5cd-4735-9199-4e508b8e3009
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:51:56Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68240c40ec901
    namespace: kserve-ci-e2e-test
    resourceVersion: "16161"
    uid: bb2a00ca-dc10-40dd-9214-84a9226ab5f2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:51:56Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68240c67792c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16163"
    uid: 5b42679f-5856-48f1-826c-9b90f21545dd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:51:56Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68240c6e07826
    namespace: kserve-ci-e2e-test
    resourceVersion: "16164"
    uid: d2896322-c4c5-4c9b-8bc2-93ff50377c04
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68241e112017f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16292"
    uid: 4b54af53-6532-4ee3-893f-c31873f211a0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68241e63690b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16298"
    uid: ac301dc1-479a-44be-a174-4d8172d7ff8d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68241e6b7838c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16300"
    uid: df68bee0-4203-4d11-805e-4bee73d83df2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68241e6c61e3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16301"
    uid: 1d8f3c6e-c9ad-46f2-9268-52e06e21d447
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68241e9ad0b63
    namespace: kserve-ci-e2e-test
    resourceVersion: "16304"
    uid: a7ad6e13-234f-433e-938e-68c7d3a974a8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d68241ea1c1423
    namespace: kserve-ci-e2e-test
    resourceVersion: "16306"
    uid: 0a9ab12e-25fb-42fb-885d-cb27539425f1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:53:28Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:52:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d682421d38ef54
    namespace: kserve-ci-e2e-test
    resourceVersion: "17052"
    uid: 1d2edcc0-adba-413e-9966-747983cc3998
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:53:26Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:26Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d6825598db6174
    namespace: kserve-ci-e2e-test
    resourceVersion: "17030"
    uid: e7ab706b-736d-498c-ae1e-0bdf7ba66e53
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:53:26Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:26Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d6825598dbfdd6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17032"
    uid: 5d279b88-40a5-49e0-9d8b-f62dfc22b724
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn
    namespace: kserve-ci-e2e-test
    resourceVersion: "16143"
    uid: 825423dc-e875-4452-837c-aa529baafdb8
  kind: Event
  lastTimestamp: "2026-09-18T19:53:28Z"
  message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp
    10.134.0.19:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:28Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8-9fltn.18d6825623a917f1
    namespace: kserve-ci-e2e-test
    resourceVersion: "17051"
    uid: 30a42409-4c58-46a9-a247-e00b71fcaee0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-585bd9bcc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15087"
    uid: 8eac0c62-bca3-453c-99dd-353ababe662d
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: 'Created pod: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk'
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8.18d6822708ed8fc4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15096"
    uid: d1cc6e6b-b8c8-46b1-9e04-ab3a045d04f9
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-585bd9bcc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15087"
    uid: 8eac0c62-bca3-453c-99dd-353ababe662d
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: 'Created pod: isvc-canary-rollback-predictor-585bd9bcc8-9d4gz'
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8.18d682270eed206a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15119"
    uid: 1da99094-80e6-412b-ae1d-c1825595a749
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-585bd9bcc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16068"
    uid: 8eac0c62-bca3-453c-99dd-353ababe662d
  kind: Event
  lastTimestamp: "2026-09-18T19:51:51Z"
  message: 'Deleted pod: isvc-canary-rollback-predictor-585bd9bcc8-5mhzk'
  metadata:
    creationTimestamp: "2026-09-18T19:51:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:51Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8.18d6823f806f8168
    namespace: kserve-ci-e2e-test
    resourceVersion: "16071"
    uid: c5c28328-1c8f-40bd-94ea-949bbda29b3c
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:56Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-585bd9bcc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16133"
    uid: 8eac0c62-bca3-453c-99dd-353ababe662d
  kind: Event
  lastTimestamp: "2026-09-18T19:51:56Z"
  message: 'Created pod: isvc-canary-rollback-predictor-585bd9bcc8-9fltn'
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-predictor-585bd9bcc8.18d68240a039c49f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16142"
    uid: fcd52f5d-07a8-4ff6-b395-bf5f0641c72d
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15086"
    uid: 68d81317-680f-4222-b962-b5111164f4f5
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-585bd9bcc8 from 0
    to 2
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-predictor.18d6822706337981
    namespace: kserve-ci-e2e-test
    resourceVersion: "15089"
    uid: 14450bd3-b686-4879-90ef-f7ddbbca0cf6
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16064"
    uid: 68d81317-680f-4222-b962-b5111164f4f5
  kind: Event
  lastTimestamp: "2026-09-18T19:51:51Z"
  message: Scaled down replica set isvc-canary-rollback-predictor-585bd9bcc8 from
    2 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:51:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:51Z"
    name: isvc-canary-rollback-predictor.18d6823f80018348
    namespace: kserve-ci-e2e-test
    resourceVersion: "16069"
    uid: e54b015d-0b1d-4184-9b39-fb334988339d
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:55Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16129"
    uid: 68d81317-680f-4222-b962-b5111164f4f5
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-585bd9bcc8 from 1
    to 2
  metadata:
    creationTimestamp: "2026-09-18T19:51:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback-predictor.18d682409c0644e9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16134"
    uid: 00303083-7cc6-4a6e-b632-73bd594526e9
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:50:05.992926Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15073"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:50:05Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:50:05Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d68227030dfb19
    namespace: kserve-ci-e2e-test
    resourceVersion: "15077"
    uid: 738fca54-f86e-4c70-b2ab-d80df4521778
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-rollback-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d682271343942e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15128"
    uid: fd587036-4967-445d-b5bf-94c9e7e4ccee
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15081"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: Add eth0 [10.134.0.17/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d682274888bed0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15160"
    uid: 6b039ddc-29f2-4338-9fb0-e1ed129f1ad6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822749d81f60
    namespace: kserve-ci-e2e-test
    resourceVersion: "15162"
    uid: a50d2177-5bdf-40b9-a37f-40f28abf24d7
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:10Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    in 2.931s (2.931s including waiting). Image size: 306813343 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:10Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d68227f88fc727
    namespace: kserve-ci-e2e-test
    resourceVersion: "15202"
    uid: a89a5af5-0f96-4399-b855-0c73292202de
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:10Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:50:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:10Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d68227fb96c332
    namespace: kserve-ci-e2e-test
    resourceVersion: "15205"
    uid: c56b9261-948b-4472-92f2-52b0e7f0313f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:10Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:50:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:10Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d68227fc02c921
    namespace: kserve-ci-e2e-test
    resourceVersion: "15207"
    uid: c58a8f21-2757-4d24-82dd-9cb75f77846e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:13Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
  metadata:
    creationTimestamp: "2026-09-18T19:50:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:13Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d68228db2ee0cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "15251"
    uid: a34db000-700c-44c1-bee3-0bfc5a5b81b2
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    in 16.211s (16.211s including waiting). Image size: 1583067017 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822ca170b65a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15385"
    uid: abe73118-6fe8-4b07-89a8-582a2a883bfa
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822ca44874d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "15388"
    uid: d747da29-8060-4e74-b248-137b8e13a59a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822ca4bab580
    namespace: kserve-ci-e2e-test
    resourceVersion: "15389"
    uid: ec655bd0-1cec-42a9-8753-8eecadea7d3b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822ca4c89dfe
    namespace: kserve-ci-e2e-test
    resourceVersion: "15390"
    uid: c4d175e8-5c16-4ad8-b525-77d39bd900b2
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.924s (2.924s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822d531cb267
    namespace: kserve-ci-e2e-test
    resourceVersion: "15420"
    uid: f6747733-976f-46ae-a012-6c969590db02
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822d58734d02
    namespace: kserve-ci-e2e-test
    resourceVersion: "15422"
    uid: db11fc66-b065-4b4c-b3cd-be19bc8d33da
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822d596ae198
    namespace: kserve-ci-e2e-test
    resourceVersion: "15423"
    uid: 9e87c9a6-ed4b-4b40-8f0c-ad842ba30a16
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:51:41Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:50:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:41Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6822dc70f256e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15966"
    uid: a7317835-7bbc-426c-b0b7-53c187feeb3d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:51:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6824098bde655
    namespace: kserve-ci-e2e-test
    resourceVersion: "16126"
    uid: 1f80a26a-f221-4876-b4ab-b9622c3e4387
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:51:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d6824098bf15ce
    namespace: kserve-ci-e2e-test
    resourceVersion: "16127"
    uid: 4e247b46-ca72-4c4e-bbe1-a0505f780685
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15075"
    uid: 0663e6db-fe95-4012-9d4f-e0e40f8e6b65
  kind: Event
  lastTimestamp: "2026-09-18T19:51:56Z"
  message: 'Readiness probe failed: Get "https://10.134.0.17:8643/healthz": dial tcp
    10.134.0.17:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:51:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:51:56Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf.18d68240a32dc51f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16152"
    uid: db7b8a33-6243-451b-be92-35c6ce580bd9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:05Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-v2-predictor-6946b55967
    namespace: kserve-ci-e2e-test
    resourceVersion: "15066"
    uid: 7acd5ccb-e7d5-469a-a8bd-8295222cb086
  kind: Event
  lastTimestamp: "2026-09-18T19:50:05Z"
  message: 'Created pod: isvc-canary-rollback-v2-predictor-6946b55967-dt4pf'
  metadata:
    creationTimestamp: "2026-09-18T19:50:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:05Z"
    name: isvc-canary-rollback-v2-predictor-6946b55967.18d68227029bc00b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15074"
    uid: bfa9b1c1-cbd0-4bc8-9377-cbd88c666fce
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:05Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15065"
    uid: 3c7ef418-730f-4bb1-895c-912691ba97b8
  kind: Event
  lastTimestamp: "2026-09-18T19:50:05Z"
  message: Scaled up replica set isvc-canary-rollback-v2-predictor-6946b55967 from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:50:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:05Z"
    name: isvc-canary-rollback-v2-predictor.18d68226fe102ff3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15067"
    uid: a266af7c-329b-435a-8a55-7a448dce0735
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 21
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "15054"
    uid: 002cfe66-0bee-429d-9f82-9a20c6e2a06d
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback.18d6822705e66ed6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16137"
    uid: 8c1aa0f2-e80a-46fc-95ee-3584bd03e6d0
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "15054"
    uid: 002cfe66-0bee-429d-9f82-9a20c6e2a06d
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: 'Failed to update status for InferenceService "isvc-canary-rollback": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback.18d682270668dc67
    namespace: kserve-ci-e2e-test
    resourceVersion: "16138"
    uid: b1f43ffa-19c4-4923-befe-5b28018681f3
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "15054"
    uid: 002cfe66-0bee-429d-9f82-9a20c6e2a06d
  kind: Event
  lastTimestamp: "2026-09-18T19:51:55Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:51:55Z"
    name: isvc-canary-rollback.18d6822706694725
    namespace: kserve-ci-e2e-test
    resourceVersion: "16139"
    uid: 6ef0a7c1-094b-46d7-8518-04ed008a0d31
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:50Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "16053"
    uid: 002cfe66-0bee-429d-9f82-9a20c6e2a06d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:18Z"
  message: InferenceService [isvc-canary-rollback] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:51:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:53:18Z"
    name: isvc-canary-rollback.18d6823f433cc8b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16949"
    uid: de6af04d-08da-41ff-9cb3-0e6767b2ac8c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:53:31.377827Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17129"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-5d5b899946-b7bt5
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d68256d4f2e771
    namespace: kserve-ci-e2e-test
    resourceVersion: "17133"
    uid: e26699ae-c3a4-4e16-a032-c9603856a60c
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d68256e2014644
    namespace: kserve-ci-e2e-test
    resourceVersion: "17145"
    uid: 44e4827d-92c0-4914-b203-dde446e7f929
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17137"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Add eth0 [10.134.0.23/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682571a8526f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17168"
    uid: 862ad71c-c043-4bcf-9538-e5bca70e5636
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682571c45479b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17177"
    uid: 1dfc79a5-5596-498e-a158-c747498bb294
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d68257209f4a8c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17181"
    uid: 9f61db1a-aa42-4931-845f-63a8d4518916
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d6825720fdbbf0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17182"
    uid: f8af226b-fffc-40c1-ad09-2b8280a83d7a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d68258179471c7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17229"
    uid: 57a57a47-57fb-4299-9ea3-48afb5a4fb4e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682581b18fa3d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17233"
    uid: ab368d6c-da83-458a-93cc-769bde75a96d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682581b7f04a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17235"
    uid: 7ffaf09f-3b52-4766-889f-2c503a6d5c9e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682581b887873
    namespace: kserve-ci-e2e-test
    resourceVersion: "17237"
    uid: 0c8f77cf-5a8c-43e4-a7c9-67b726c8c672
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682581e61c00e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17241"
    uid: 7fc290eb-7f56-4b24-ba7e-86eb3218e9c6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682581ec452a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17243"
    uid: 4febe51d-498a-461c-819c-6638c74fcc71
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:54:44Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:44Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d682588f931c8c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18009"
    uid: 7b70eb79-b9c8-4dff-b381-46addf18bbd7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:54:55Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:54:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:55Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d6826a534d5df7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18121"
    uid: a2221e60-e338-40ff-9277-83d95e1a6979
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17131"
    uid: 80df351c-06a1-4b45-9a4e-ea78e9296626
  kind: Event
  lastTimestamp: "2026-09-18T19:54:55Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:54:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:55Z"
    name: isvc-canary-stop-predictor-5d5b899946-b7bt5.18d6826a534df51c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18122"
    uid: 3f03b24f-db35-41a5-8523-1cb8cdcea530
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:53:31.321689Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17115"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-5d5b899946-mhxmz
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68256d19a4a9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17121"
    uid: 71375a3c-8ecf-43c6-b5b6-1a2440e77213
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68256df3b30e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17143"
    uid: c2d97d2d-f351-41aa-ae68-23c15d1a4347
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17127"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Add eth0 [10.133.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d682571747917b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17161"
    uid: b89eae0e-61dd-40ef-9902-a1c8e671664c
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d6825718cc4ea7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17164"
    uid: 80ba3f3b-4832-452f-a287-4ba6d42ee2ea
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d682571b42b3c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17171"
    uid: 563f1da9-bae4-4e60-b15c-04d4e977334a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d682571ba332a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17175"
    uid: 82a67df0-0571-4528-9e6b-d06ebd80cc5c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68257f5ef7c9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17219"
    uid: 134e6115-28a6-42f8-804f-a9fbfbd0820f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68257f89ddd48
    namespace: kserve-ci-e2e-test
    resourceVersion: "17221"
    uid: 4883a17d-dd5b-4a3b-8351-816b935e370a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68257f91e71fd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17222"
    uid: b92641c3-0393-4534-80e4-71525eff2a52
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68257f9270faf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17223"
    uid: 8e0a9222-5676-4717-8178-68a11d0773b8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68257fbc5304e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17225"
    uid: b53bc762-c6a3-4782-a66a-c4cd830f4095
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d68257fc20bdcf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17226"
    uid: 91456108-a230-446c-a511-127808e59298
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:55:03Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:03Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d6825831f5575c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18217"
    uid: 91bbe6cb-a55f-43cd-96fa-620523e46d33
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:55:01Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:01Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d6826bcf72d698
    namespace: kserve-ci-e2e-test
    resourceVersion: "18199"
    uid: a82a20a9-2b4b-420d-83f9-66693efac458
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:55:01Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:01Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d6826bcf738eae
    namespace: kserve-ci-e2e-test
    resourceVersion: "18200"
    uid: 4acba461-6f35-4211-b7fc-33b193ad1206
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 310f5d1e-6c8b-47c0-aaf2-b8b04c6eed54
  kind: Event
  lastTimestamp: "2026-09-18T19:55:03Z"
  message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp
    10.133.0.29:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:55:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:03Z"
    name: isvc-canary-stop-predictor-5d5b899946-mhxmz.18d6826c383c3e56
    namespace: kserve-ci-e2e-test
    resourceVersion: "18216"
    uid: 777849e0-0941-4368-9a71-4993d9345a92
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-5d5b899946
    namespace: kserve-ci-e2e-test
    resourceVersion: "17104"
    uid: 6d451b5c-efb4-444a-801f-f82282161e8a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: 'Created pod: isvc-canary-stop-predictor-5d5b899946-mhxmz'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor-5d5b899946.18d68256d131dd3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17117"
    uid: aafcb1d3-14c7-4bb6-9ea4-2b4ad4b61524
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-5d5b899946
    namespace: kserve-ci-e2e-test
    resourceVersion: "17104"
    uid: 6d451b5c-efb4-444a-801f-f82282161e8a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: 'Created pod: isvc-canary-stop-predictor-5d5b899946-b7bt5'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor-5d5b899946.18d68256d486af86
    namespace: kserve-ci-e2e-test
    resourceVersion: "17130"
    uid: 81df78cb-a7ec-4903-bb47-975172ab04ca
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:54Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-5d5b899946
    namespace: kserve-ci-e2e-test
    resourceVersion: "18103"
    uid: 6d451b5c-efb4-444a-801f-f82282161e8a
  kind: Event
  lastTimestamp: "2026-09-18T19:54:54Z"
  message: 'Deleted pod: isvc-canary-stop-predictor-5d5b899946-b7bt5'
  metadata:
    creationTimestamp: "2026-09-18T19:54:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:54:54Z"
    name: isvc-canary-stop-predictor-5d5b899946.18d6826a49644291
    namespace: kserve-ci-e2e-test
    resourceVersion: "18107"
    uid: 4ae58afb-3b08-4aa1-bf5d-4b1d0daef0e9
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17103"
    uid: a60f49f1-2fb0-494a-9675-d0142bffb39c
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: Scaled up replica set isvc-canary-stop-predictor-5d5b899946 from 0 to 2
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-predictor.18d68256ce249137
    namespace: kserve-ci-e2e-test
    resourceVersion: "17106"
    uid: 83872ddc-405b-4e95-bb87-c4f2f3f7b488
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:54Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18099"
    uid: a60f49f1-2fb0-494a-9675-d0142bffb39c
  kind: Event
  lastTimestamp: "2026-09-18T19:54:54Z"
  message: Scaled down replica set isvc-canary-stop-predictor-5d5b899946 from 2 to
    1
  metadata:
    creationTimestamp: "2026-09-18T19:54:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:54:54Z"
    name: isvc-canary-stop-predictor.18d6826a4900ce48
    namespace: kserve-ci-e2e-test
    resourceVersion: "18105"
    uid: 10acc2c4-60c7-46ab-bed3-7c3c232ce4a2
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:53:31.308903Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17111"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d68256d0d7dd56
    namespace: kserve-ci-e2e-test
    resourceVersion: "17116"
    uid: a6c115a5-cc46-4a6f-b84b-661daa2ac528
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d68256e205eaf4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17146"
    uid: 6f7c0f69-f8aa-4cf9-ad23-fc6356a98823
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17126"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Add eth0 [10.134.0.22/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d6825716abb7e1
    namespace: kserve-ci-e2e-test
    resourceVersion: "17159"
    uid: 8d9abf52-b565-40eb-a6a0-f3aecbd0a0a6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d68257180003b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "17163"
    uid: 3f96192a-e1f5-4b8c-9aff-02d83580720b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682571ba75ad6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17174"
    uid: fc1ab56d-4bf2-4bf0-a437-463ce6388ee9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682571c292ab6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17176"
    uid: 3a493180-54e9-4376-b82e-c334f81e8465
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d6825817ab44ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "17230"
    uid: 66a5015a-e301-4296-8079-676bea6a4720
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682581b264edf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17234"
    uid: e2753a8b-97ef-424d-b0d3-393e37d8eebf
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682581b8aade2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17239"
    uid: 4fa04e82-a163-4db5-8e5b-d50f106d6d72
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682581b959199
    namespace: kserve-ci-e2e-test
    resourceVersion: "17240"
    uid: a8783d98-09bb-472c-990b-f27e1e510e06
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682581e6632b7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17242"
    uid: 87bdbed7-8d59-47a3-a62a-46c2dcba5ffc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:36Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:36Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682581ed09796
    namespace: kserve-ci-e2e-test
    resourceVersion: "17244"
    uid: 3867f1bb-b5ac-470c-b1d9-9f53364b690b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:55:04Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:04Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d682588f939741
    namespace: kserve-ci-e2e-test
    resourceVersion: "18234"
    uid: ab0b4e79-b9c5-4fa9-8e04-952066dfdc75
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:55:01Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:01Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d6826bca08ab6b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18189"
    uid: d029fc3a-107d-4a26-8dcd-cc2407c95729
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:55:01Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:01Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d6826bca096b9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18194"
    uid: 09d6ad4c-e1cf-4d22-be89-6f147effd067
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht
    namespace: kserve-ci-e2e-test
    resourceVersion: "17113"
    uid: 4ae1660d-14f3-449d-9be0-f0cfa8fa142a
  kind: Event
  lastTimestamp: "2026-09-18T19:55:04Z"
  message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp
    10.134.0.22:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:55:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:04Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht.18d6826c95d2310b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18233"
    uid: 5111b65c-45fc-460c-9b9e-54062517b910
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-v2-predictor-5f87c564d7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17091"
    uid: 24521791-d7b0-43a5-b4df-3e65995e7aca
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: 'Created pod: isvc-canary-stop-v2-predictor-5f87c564d7-c88ht'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-v2-predictor-5f87c564d7.18d68256d0762823
    namespace: kserve-ci-e2e-test
    resourceVersion: "17112"
    uid: 2e1afdea-e9ba-4b07-ac1e-39d2633ba717
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17089"
    uid: 166f96df-e377-4bd9-82fa-2aabca897f87
  kind: Event
  lastTimestamp: "2026-09-18T19:53:31Z"
  message: Scaled up replica set isvc-canary-stop-v2-predictor-5f87c564d7 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:31Z"
    name: isvc-canary-stop-v2-predictor.18d68256cbbcd48f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17093"
    uid: 55f84e41-91c1-4a74-9f8f-463369595aec
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 21
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "17082"
    uid: 5d81aee2-7bca-4938-8961-a23a4d226227
  kind: Event
  lastTimestamp: "2026-09-18T19:54:55Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:54:55Z"
    name: isvc-canary-stop.18d68256cdcd06bc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18120"
    uid: 681aa717-8841-454f-ba72-845956306a6d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "17082"
    uid: 5d81aee2-7bca-4938-8961-a23a4d226227
  kind: Event
  lastTimestamp: "2026-09-18T19:54:54Z"
  message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:54:54Z"
    name: isvc-canary-stop.18d68256ce408467
    namespace: kserve-ci-e2e-test
    resourceVersion: "18116"
    uid: 07dc47a6-6f3a-480e-9fbf-52bc87d14823
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:31Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "17082"
    uid: 5d81aee2-7bca-4938-8961-a23a4d226227
  kind: Event
  lastTimestamp: "2026-09-18T19:54:54Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:53:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:54:54Z"
    name: isvc-canary-stop.18d68256ce40c251
    namespace: kserve-ci-e2e-test
    resourceVersion: "18117"
    uid: f283be01-53b1-4dd4-b714-f668814a41bf
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:54Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: 5d81aee2-7bca-4938-8961-a23a4d226227
  kind: Event
  lastTimestamp: "2026-09-18T19:54:54Z"
  message: InferenceService [isvc-canary-stop] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:54:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:54:54Z"
    name: isvc-canary-stop.18d6826a48da72c5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18104"
    uid: 18483e61-b8a0-4e4e-9b7d-1f2133be8796
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:57:20.109177Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19708"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:57:20Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828c16665067
    namespace: kserve-ci-e2e-test
    resourceVersion: "19714"
    uid: ec83aa94-3cc7-441f-a90d-b9aec487f28a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:20Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19711"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:20Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-c46f26-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:20Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828c201b25e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "19729"
    uid: 26ad03ad-1a87-4a09-9681-e69f4fe6e9ef
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:22Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19718"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:22Z"
  message: Add eth0 [10.134.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:57:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:57:22Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828c91af2f69
    namespace: kserve-ci-e2e-test
    resourceVersion: "19740"
    uid: af38b8e8-ed1f-471f-86e8-99ca316fb83d
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19711"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:25Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:25Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828c931a3dcc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19777"
    uid: 5fab7cf2-5826-422c-92f7-c99172d9b312
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19711"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:25Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:57:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:25Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828c95cc4f03
    namespace: kserve-ci-e2e-test
    resourceVersion: "19779"
    uid: 0b43059c-ac98-46df-a596-0b7cf4d47062
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19711"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:25Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:57:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:25Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828c963af70b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19782"
    uid: 73153b3d-1f11-4113-bb5c-f4198b1e66aa
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19711"
    uid: d9f0f324-d328-4c4b-9646-b3011589817c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:29Z"
  message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-c46f26-predictor-688947fcf7-2qj57_kserve-ci-e2e-test(d9f0f324-d328-4c4b-9646-b3011589817c)
  metadata:
    creationTimestamp: "2026-09-18T19:57:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:29Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57.18d6828e53a33a1d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19902"
    uid: 53ebf763-49f8-4feb-a170-63b659d2e1e8
  reason: BackOff
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:20Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-init-fail-c46f26-predictor-688947fcf7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19701"
    uid: 5c034221-2145-450b-a4d9-2d76e10cdb09
  kind: Event
  lastTimestamp: "2026-09-18T19:57:20Z"
  message: 'Created pod: isvc-init-fail-c46f26-predictor-688947fcf7-2qj57'
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:20Z"
    name: isvc-init-fail-c46f26-predictor-688947fcf7.18d6828c15daca0b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19710"
    uid: 64dfb57c-802a-472c-b3b3-1bc68499bbdd
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:20Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-init-fail-c46f26-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19699"
    uid: 04314e85-a688-4d88-a06b-cf950dc4ef34
  kind: Event
  lastTimestamp: "2026-09-18T19:57:20Z"
  message: Scaled up replica set isvc-init-fail-c46f26-predictor-688947fcf7 from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:20Z"
    name: isvc-init-fail-c46f26-predictor.18d6828c12efe55b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19702"
    uid: 7910a2f8-732f-4e3c-a4fb-a221aec8ae54
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-c46f26
    namespace: kserve-ci-e2e-test
    resourceVersion: "19692"
    uid: ee430766-ca6b-41e8-872f-2a8fe2ce5283
  kind: Event
  lastTimestamp: "2026-09-18T19:57:29Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:29Z"
    name: isvc-init-fail-c46f26.18d6828c13202fa9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19910"
    uid: 4922f22c-70bd-4a59-b0c9-41b0e421d642
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-c46f26
    namespace: kserve-ci-e2e-test
    resourceVersion: "19692"
    uid: ee430766-ca6b-41e8-872f-2a8fe2ce5283
  kind: Event
  lastTimestamp: "2026-09-18T19:57:20Z"
  message: 'Failed to update status for InferenceService "isvc-init-fail-c46f26":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-c46f26":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:20Z"
    name: isvc-init-fail-c46f26.18d6828c13818207
    namespace: kserve-ci-e2e-test
    resourceVersion: "19705"
    uid: 32069151-0748-47fc-b2cb-dd2e8874397f
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-c46f26
    namespace: kserve-ci-e2e-test
    resourceVersion: "19692"
    uid: ee430766-ca6b-41e8-872f-2a8fe2ce5283
  kind: Event
  lastTimestamp: "2026-09-18T19:57:20Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-init-fail-c46f26": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:57:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:20Z"
    name: isvc-init-fail-c46f26.18d6828c1381df62
    namespace: kserve-ci-e2e-test
    resourceVersion: "19707"
    uid: 52680ac0-b2cd-4e28-823b-78c3575437bf
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:55:51.770480Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18848"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:55:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:55:51Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d682778500bdd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18855"
    uid: a2453d2d-9ce8-4654-9b26-a8563981254d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18860"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:52Z"
  message: Add eth0 [10.133.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:55:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:55:52Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68277a744685f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18867"
    uid: 1256c74e-5b02-4b6d-8a30-955a2ca82b79
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:52Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68277a8ae5306
    namespace: kserve-ci-e2e-test
    resourceVersion: "18869"
    uid: fa38d327-c11d-4945-8145-b761793b1938
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:55:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:52Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68277ab40862c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18871"
    uid: 7144be3c-c0a6-47e4-a256-00a9443c0193
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:55:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:52Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68277aba51b17
    namespace: kserve-ci-e2e-test
    resourceVersion: "18872"
    uid: ef0854af-c358-4cbd-9760-a6eb5a85e1c5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:56Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278b92d389b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18920"
    uid: 122f5c1b-e4aa-4ad5-abe2-af7804f1642a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:55:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:56Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278bbed74be
    namespace: kserve-ci-e2e-test
    resourceVersion: "18925"
    uid: 4a2ca484-37aa-4ff0-8102-3ed4f6d8efd8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:56Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:56Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278bc52fae3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18926"
    uid: b201dd9b-b57f-4fd8-9163-bd621b8b6dfe
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:56Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:56Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278bc5dd2e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18928"
    uid: b9e33ebb-6990-4fde-b70b-d6d0ea636784
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:57Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:55:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:57Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278bf231911
    namespace: kserve-ci-e2e-test
    resourceVersion: "18930"
    uid: b67bb389-3f8c-4159-aafb-053e5c62ce6d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:57Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:57Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278bf8a42c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18932"
    uid: 91ce6c2e-8c1b-40ed-a9ab-c99201ec7a95
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:57Z"
  message: Container image "quay.io/opendatahub/kserve-agent@sha256:ee489751b997752a8855e389ed75068cf003229489532801b9b57f9452d3ed82"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:57Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278bf95f272
    namespace: kserve-ci-e2e-test
    resourceVersion: "18933"
    uid: 9fd39ca7-8bab-4efb-ae84-702f4bfe04b4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:57Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-18T19:55:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:57Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278c258f8e2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18934"
    uid: 23f6a3df-7ac4-4690-9b3d-d35bedf17160
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:55:57Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-18T19:55:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:57Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278c2e27c44
    namespace: kserve-ci-e2e-test
    resourceVersion: "18935"
    uid: 44421187-e1b1-4bae-a004-8915acf53a32
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:57:34Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:55:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:34Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d68278f53d4994
    namespace: kserve-ci-e2e-test
    resourceVersion: "19980"
    uid: 840d6485-a2c0-40c5-b2ae-095593932567
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:57:34Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-18T19:55:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:34Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d682793112b4df
    namespace: kserve-ci-e2e-test
    resourceVersion: "19981"
    uid: 71835335-fab5-4614-8a80-5698ad36653c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:57:26Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:26Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d6828dac9484d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19816"
    uid: be356c5b-0f59-4ac9-8b22-031ad11df87e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:57:26Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-18T19:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:26Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d6828dac94d79a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19817"
    uid: 8e9edfcb-9308-4f61-91ce-a02b952c363e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:57:26Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:26Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d6828dac94f4ab
    namespace: kserve-ci-e2e-test
    resourceVersion: "19818"
    uid: d02367e5-824e-4de9-baa7-60737b166b29
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs
    namespace: kserve-ci-e2e-test
    resourceVersion: "18852"
    uid: 15831d0b-ce18-4541-8c7d-d2995e430b89
  kind: Event
  lastTimestamp: "2026-09-18T19:57:44Z"
  message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp
    10.133.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:57:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:44Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs.18d6828e615b3faf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20117"
    uid: 2cdc1f53-bf91-46cb-ba62-c252c81d868a
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-91b5b-predictor-578588fc8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18838"
    uid: 43dabcbe-571a-4a15-8a97-513380e16780
  kind: Event
  lastTimestamp: "2026-09-18T19:55:51Z"
  message: 'Created pod: isvc-logger-raw-91b5b-predictor-578588fc8f-rnhhs'
  metadata:
    creationTimestamp: "2026-09-18T19:55:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:51Z"
    name: isvc-logger-raw-91b5b-predictor-578588fc8f.18d68277847a6e59
    namespace: kserve-ci-e2e-test
    resourceVersion: "18851"
    uid: a955c5e1-8215-4712-b802-cec71872d5aa
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18836"
    uid: a7eb411f-b8ae-4dd0-bea0-dbcba081416f
  kind: Event
  lastTimestamp: "2026-09-18T19:55:51Z"
  message: Scaled up replica set isvc-logger-raw-91b5b-predictor-578588fc8f from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-18T19:55:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:51Z"
    name: isvc-logger-raw-91b5b-predictor.18d68277810d0f36
    namespace: kserve-ci-e2e-test
    resourceVersion: "18840"
    uid: f59c224c-48a3-4630-8150-f11154edfd80
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18837"
    uid: 8c3234ec-a46b-4105-9327-9706470d416b
  kind: Event
  lastTimestamp: "2026-09-18T19:56:36Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:56:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:36Z"
    name: isvc-logger-raw-91b5b-predictor.18d6827aff90542f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19252"
    uid: 90ff541b-d752-4a39-aaa2-19f20af484ac
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18837"
    uid: 8c3234ec-a46b-4105-9327-9706470d416b
  kind: Event
  lastTimestamp: "2026-09-18T19:56:36Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:56:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:36Z"
    name: isvc-logger-raw-91b5b-predictor.18d6827afffa84f5
    namespace: kserve-ci-e2e-test
    resourceVersion: "19253"
    uid: 5344fabb-35bd-46f8-90e6-1a90778150c2
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:51Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19017"
    uid: 8c3234ec-a46b-4105-9327-9706470d416b
  kind: Event
  lastTimestamp: "2026-09-18T19:57:21Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:56:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:21Z"
    name: isvc-logger-raw-91b5b-predictor.18d682857d22db44
    namespace: kserve-ci-e2e-test
    resourceVersion: "19737"
    uid: d8b6d1d8-9f5b-4822-97d4-442a58ee94ba
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:51Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19017"
    uid: 8c3234ec-a46b-4105-9327-9706470d416b
  kind: Event
  lastTimestamp: "2026-09-18T19:57:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:56:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:21Z"
    name: isvc-logger-raw-91b5b-predictor.18d682857dd8922c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19738"
    uid: 228f3446-566f-4f66-abfa-344d0468fb6f
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-91b5b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18829"
    uid: 59f81478-c2e4-46d4-804c-c68b976770e7
  kind: Event
  lastTimestamp: "2026-09-18T19:57:15Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:55:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:15Z"
    name: isvc-logger-raw-91b5b.18d6827781228b19
    namespace: kserve-ci-e2e-test
    resourceVersion: "19644"
    uid: 8e0e5c23-c6e1-4c9a-ad31-99d9fa7f44fd
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-91b5b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18829"
    uid: 59f81478-c2e4-46d4-804c-c68b976770e7
  kind: Event
  lastTimestamp: "2026-09-18T19:55:51Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-91b5b":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-91b5b":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:55:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:51Z"
    name: isvc-logger-raw-91b5b.18d6827781b74e32
    namespace: kserve-ci-e2e-test
    resourceVersion: "18844"
    uid: 598d4a9d-5af7-4b2a-b3a3-de5047eca111
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:15Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-91b5b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19641"
    uid: 59f81478-c2e4-46d4-804c-c68b976770e7
  kind: Event
  lastTimestamp: "2026-09-18T19:57:15Z"
  message: InferenceService [isvc-logger-raw-91b5b] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:57:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:15Z"
    name: isvc-logger-raw-91b5b.18d6828ae9f0a9f2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19642"
    uid: 10dbceea-a5bb-46c1-8368-941caf084156
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:05:11.886841Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23940"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:05:11Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:05:11Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682f9ee8a2f67
    namespace: kserve-ci-e2e-test
    resourceVersion: "23945"
    uid: b57caf43-1505-462e-8985-f2effe62b100
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23950"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:12Z"
  message: Add eth0 [10.134.0.36/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:05:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:05:12Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fa10526a6e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23959"
    uid: 1b538e4e-cec2-4a28-a10b-342032d6c47e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:12Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:05:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:12Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fa11ac612d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23961"
    uid: 7727b0fd-0856-40eb-8685-15ad37d54838
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:12Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T20:05:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:12Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fa148be044
    namespace: kserve-ci-e2e-test
    resourceVersion: "23962"
    uid: 86384556-3612-4633-9876-f44546a6964b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:12Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T20:05:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:12Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fa14f8f733
    namespace: kserve-ci-e2e-test
    resourceVersion: "23963"
    uid: 624512af-b791-4399-83ed-ce495db3fdc3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb03875a27
    namespace: kserve-ci-e2e-test
    resourceVersion: "23994"
    uid: 7043d2c4-7a98-4172-bb92-02bb8c5d54bb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb064d77bd
    namespace: kserve-ci-e2e-test
    resourceVersion: "23997"
    uid: 28276d67-1296-486d-8835-64229c4c009a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb06d1037e
    namespace: kserve-ci-e2e-test
    resourceVersion: "24000"
    uid: b4cb319c-1230-4c67-94d8-d767b5e4933b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb06da87d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24001"
    uid: a93e4176-2bb7-43d0-8c57-7c0995675881
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb097af779
    namespace: kserve-ci-e2e-test
    resourceVersion: "24003"
    uid: 7c71b12e-5fb2-444d-9fe9-e4305a81873e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb09ea804c
    namespace: kserve-ci-e2e-test
    resourceVersion: "24004"
    uid: 52b845a1-b9ec-4d36-9427-6c9c73dae22f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:16Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:ee489751b997752a8855e389ed75068cf003229489532801b9b57f9452d3ed82"
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:16Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb09f72650
    namespace: kserve-ci-e2e-test
    resourceVersion: "24005"
    uid: bf7e0f5c-9fa4-48a1-ae29-245fab5e2553
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:19Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:ee489751b997752a8855e389ed75068cf003229489532801b9b57f9452d3ed82"
    in 2.443s (2.443s including waiting). Image size: 243360834 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T20:05:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:19Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb9b96898e
    namespace: kserve-ci-e2e-test
    resourceVersion: "24021"
    uid: 3e6e8164-45c5-439e-b357-d35f649facb7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:19Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-18T20:05:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:19Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb9ecd6f47
    namespace: kserve-ci-e2e-test
    resourceVersion: "24022"
    uid: 8efcc2b5-5ad7-47c2-a2fc-8c92a28cc2d6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:05:19Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-18T20:05:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:19Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fb9f85b616
    namespace: kserve-ci-e2e-test
    resourceVersion: "24023"
    uid: 17a89bd1-e9bc-4c6a-92c8-611710743a85
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:07:02Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T20:05:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:07:02Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fbb7781583
    namespace: kserve-ci-e2e-test
    resourceVersion: "24838"
    uid: 5bf0bbc6-38c4-46df-b059-d9f707e33495
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:06:52Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-18T20:05:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:06:52Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d682fbf34a7cd7
    namespace: kserve-ci-e2e-test
    resourceVersion: "24763"
    uid: a60c770d-c858-4fa1-8fbc-23bba6ff9c39
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:06:47Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:06:47Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d683101cf3e646
    namespace: kserve-ci-e2e-test
    resourceVersion: "24666"
    uid: 8433b3a1-e773-4a8a-a1ea-ce52a38bb494
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:06:47Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-18T20:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:06:47Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d683101cf47bc3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24669"
    uid: 30235666-457f-4d22-9d3c-3b452de0e04b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:06:47Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:06:47Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d683101cf4bd9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "24672"
    uid: 631aa829-477a-453c-bf4c-d880e33c924d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs
    namespace: kserve-ci-e2e-test
    resourceVersion: "23942"
    uid: cde13c35-a2b1-4644-bda9-2e0bb5c223fe
  kind: Event
  lastTimestamp: "2026-09-18T20:07:01Z"
  message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp
    10.134.0.36:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:07:01Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594-9swqs.18d683112380f7e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "24830"
    uid: cb41607f-9685-4c7d-9040-8790d4ebed5d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-cipn-predictor-6b79574594
    namespace: kserve-ci-e2e-test
    resourceVersion: "23930"
    uid: 412686bb-13c4-421b-82dd-8b3afb71b261
  kind: Event
  lastTimestamp: "2026-09-18T20:05:11Z"
  message: 'Created pod: isvc-logger-raw-cipn-predictor-6b79574594-9swqs'
  metadata:
    creationTimestamp: "2026-09-18T20:05:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:11Z"
    name: isvc-logger-raw-cipn-predictor-6b79574594.18d682f9ee2abc6a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23943"
    uid: 7e90c81a-ec01-4afb-92c5-b11ac8cc3485
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23928"
    uid: 554dfd63-da31-4307-86be-0c106b8d27c3
  kind: Event
  lastTimestamp: "2026-09-18T20:05:11Z"
  message: Scaled up replica set isvc-logger-raw-cipn-predictor-6b79574594 from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-18T20:05:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:11Z"
    name: isvc-logger-raw-cipn-predictor.18d682f9eae980d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "23932"
    uid: cdf6f21d-6548-403b-a2c1-8ca7821e00cd
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23929"
    uid: bcae227b-59c4-45f0-a240-17b970fe3cdd
  kind: Event
  lastTimestamp: "2026-09-18T20:05:56Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T20:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:56Z"
    name: isvc-logger-raw-cipn-predictor.18d682fd69b07a83
    namespace: kserve-ci-e2e-test
    resourceVersion: "24290"
    uid: a17ba399-2b49-4483-8555-b0b71bd7b210
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23929"
    uid: bcae227b-59c4-45f0-a240-17b970fe3cdd
  kind: Event
  lastTimestamp: "2026-09-18T20:05:56Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T20:05:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:56Z"
    name: isvc-logger-raw-cipn-predictor.18d682fd6a0dd51f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24291"
    uid: b112640f-1960-4968-9b88-b93cbd1dd5cd
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24083"
    uid: bcae227b-59c4-45f0-a240-17b970fe3cdd
  kind: Event
  lastTimestamp: "2026-09-18T20:06:41Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T20:06:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:06:41Z"
    name: isvc-logger-raw-cipn-predictor.18d68307e67aab3e
    namespace: kserve-ci-e2e-test
    resourceVersion: "24603"
    uid: 8f988ec4-0df8-4d0b-9df9-72b2f867cbca
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "24083"
    uid: bcae227b-59c4-45f0-a240-17b970fe3cdd
  kind: Event
  lastTimestamp: "2026-09-18T20:06:41Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T20:06:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:06:41Z"
    name: isvc-logger-raw-cipn-predictor.18d68307e6de0708
    namespace: kserve-ci-e2e-test
    resourceVersion: "24604"
    uid: c606cb0a-3f4e-40d6-b09a-27abdefe0323
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23922"
    uid: de717884-47de-4d88-a964-af793e3e62f3
  kind: Event
  lastTimestamp: "2026-09-18T20:06:32Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T20:05:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:06:32Z"
    name: isvc-logger-raw-cipn.18d682f9eaeed881
    namespace: kserve-ci-e2e-test
    resourceVersion: "24542"
    uid: a103d9d8-156d-4b9b-a55f-5713dfad1336
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23922"
    uid: de717884-47de-4d88-a964-af793e3e62f3
  kind: Event
  lastTimestamp: "2026-09-18T20:05:11Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-cipn": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T20:05:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:05:11Z"
    name: isvc-logger-raw-cipn.18d682f9eb8e554b
    namespace: kserve-ci-e2e-test
    resourceVersion: "23936"
    uid: a4dd75cb-9368-4f8e-a40a-60f2ac592d58
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24540"
    uid: de717884-47de-4d88-a964-af793e3e62f3
  kind: Event
  lastTimestamp: "2026-09-18T20:06:32Z"
  message: InferenceService [isvc-logger-raw-cipn] is Ready
  metadata:
    creationTimestamp: "2026-09-18T20:06:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:06:32Z"
    name: isvc-logger-raw-cipn.18d6830cbd486b1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "24541"
    uid: dfc3f513-435a-4600-bdb9-1f20b4c3b1d1
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:55:11.594605Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18317"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-primary-81daee-predictor-865f44d78-xlh89
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:55:11Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826e2a5588f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18324"
    uid: 6b49f0dc-e337-4f1f-9d00-c7e56c04b7b5
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:11Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-81daee-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:11Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826e320c9268
    namespace: kserve-ci-e2e-test
    resourceVersion: "18333"
    uid: f365d9b2-23a4-4a6a-ab62-7cb7bdd4d7f6
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18327"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:12Z"
  message: Add eth0 [10.134.0.24/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:55:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:55:12Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826e701bd44e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18347"
    uid: 66e9aadf-3d2d-4b59-a455-013a4cf85e5e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:12Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:12Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826e71ab8c69
    namespace: kserve-ci-e2e-test
    resourceVersion: "18350"
    uid: 8fa2aa94-7fb3-4489-bd68-b7a902dd1065
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:12Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:55:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:12Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826e742d7ad8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18353"
    uid: a1c911f3-0f7f-4a8d-a018-fe517608ad2f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:12Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:55:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:12Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826e749aa732
    namespace: kserve-ci-e2e-test
    resourceVersion: "18354"
    uid: 645afb88-7264-40e1-aa11-ede10a5a9bbb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:17Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:17Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826f786cd4a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18398"
    uid: b5b15c19-6802-4bf8-a92f-98fd7eddb68c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:17Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:55:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:17Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826f7b7a342d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18401"
    uid: 564f73fd-c868-40ca-9268-6384d63cd545
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:17Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:17Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826f7be9ddf2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18402"
    uid: 18c05758-b13f-4d30-aa7a-19541830e9e9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:17Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:17Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826f7bf3a7e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18403"
    uid: 97bca50d-3e46-490f-85ad-caec06e5604a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:17Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:55:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:17Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826f7eac28a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18406"
    uid: 1a965c00-f903-4805-8fe6-3ea0d723ed58
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:55:17Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:17Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826f7f18a673
    namespace: kserve-ci-e2e-test
    resourceVersion: "18408"
    uid: dff9f773-96b2-4611-9ad8-82c0bdc5e67b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:57:15Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:55:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:15Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6826ff04fe86e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19646"
    uid: 1c80361f-5f38-48c3-8da6-353abfeacf5c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:57:14Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:57:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:14Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6828ae0ce0b57
    namespace: kserve-ci-e2e-test
    resourceVersion: "19620"
    uid: 638e5663-ee34-49ba-8cce-944ef871c0aa
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:14Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:57:14Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:57:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:14Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6828ae0ce85a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19622"
    uid: 321ead26-1758-489c-8883-597898e2f346
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-81daee-predictor-865f44d78-xlh89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18319"
    uid: da54697d-7fcb-44ad-a513-7e4a13b26c58
  kind: Event
  lastTimestamp: "2026-09-18T19:57:15Z"
  message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp
    10.134.0.24:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:57:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:15Z"
    name: isvc-primary-81daee-predictor-865f44d78-xlh89.18d6828af2aa1d6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19645"
    uid: 7770802b-8d5f-4a55-b111-e917e62f1ed6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-primary-81daee-predictor-865f44d78
    namespace: kserve-ci-e2e-test
    resourceVersion: "18310"
    uid: 71e8dd2a-79a0-41c2-a771-04f3cd928187
  kind: Event
  lastTimestamp: "2026-09-18T19:55:11Z"
  message: 'Created pod: isvc-primary-81daee-predictor-865f44d78-xlh89'
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:11Z"
    name: isvc-primary-81daee-predictor-865f44d78.18d6826e29d63b6c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18318"
    uid: 118adbf1-d2f9-45af-b6a6-7d39f41526e1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-primary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18308"
    uid: 99967226-ffd7-4a7f-b131-30c40b78b2f8
  kind: Event
  lastTimestamp: "2026-09-18T19:55:11Z"
  message: Scaled up replica set isvc-primary-81daee-predictor-865f44d78 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:11Z"
    name: isvc-primary-81daee-predictor.18d6826e269b69ee
    namespace: kserve-ci-e2e-test
    resourceVersion: "18311"
    uid: 2ea06867-2582-4e36-8bb5-7484e75ac061
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18309"
    uid: 625914e2-f3e1-48b0-8776-f274b03bb193
  kind: Event
  lastTimestamp: "2026-09-18T19:55:56Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:55:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:56Z"
    name: isvc-primary-81daee-predictor.18d68271a5288f8c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18914"
    uid: ca02b4eb-da0e-4903-a11e-b33e99890445
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18309"
    uid: 625914e2-f3e1-48b0-8776-f274b03bb193
  kind: Event
  lastTimestamp: "2026-09-18T19:55:56Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:55:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:56Z"
    name: isvc-primary-81daee-predictor.18d68271a5a0cd29
    namespace: kserve-ci-e2e-test
    resourceVersion: "18915"
    uid: 50e8f2ac-ebb9-4d88-bb36-28d62ba927bf
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18486"
    uid: 625914e2-f3e1-48b0-8776-f274b03bb193
  kind: Event
  lastTimestamp: "2026-09-18T19:57:11Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:56:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:11Z"
    name: isvc-primary-81daee-predictor.18d6827c222c367e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19569"
    uid: a7fc6e81-1d2a-4ac4-b4d9-b83a9153a8e9
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18486"
    uid: 625914e2-f3e1-48b0-8776-f274b03bb193
  kind: Event
  lastTimestamp: "2026-09-18T19:57:11Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:56:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:11Z"
    name: isvc-primary-81daee-predictor.18d6827c22954f06
    namespace: kserve-ci-e2e-test
    resourceVersion: "19570"
    uid: 2867b8ff-eabe-40dc-bb63-e80cb1a1683f
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 13
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "18300"
    uid: 8402e37c-051f-44a4-b158-a4292824070f
  kind: Event
  lastTimestamp: "2026-09-18T19:56:35Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:56:35Z"
    name: isvc-primary-81daee.18d6826e269e845b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19242"
    uid: de8d66f0-e97a-4695-ad4d-d9ca73bcf07d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "18300"
    uid: 8402e37c-051f-44a4-b158-a4292824070f
  kind: Event
  lastTimestamp: "2026-09-18T19:55:11Z"
  message: 'Failed to update status for InferenceService "isvc-primary-81daee": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-81daee":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:11Z"
    name: isvc-primary-81daee.18d6826e2742123d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18329"
    uid: 947e4ad8-2979-4b96-9c40-35ca6b078033
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "18300"
    uid: 8402e37c-051f-44a4-b158-a4292824070f
  kind: Event
  lastTimestamp: "2026-09-18T19:55:11Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-primary-81daee": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:55:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:11Z"
    name: isvc-primary-81daee.18d6826e27424fe0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18330"
    uid: 41eaca53-b1bd-45c4-a328-40c72b1ba9c8
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:35Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "19240"
    uid: 8402e37c-051f-44a4-b158-a4292824070f
  kind: Event
  lastTimestamp: "2026-09-18T19:56:35Z"
  message: InferenceService [isvc-primary-81daee] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:56:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:56:35Z"
    name: isvc-primary-81daee.18d68281a78d3614
    namespace: kserve-ci-e2e-test
    resourceVersion: "19241"
    uid: 2f0456e5-5b56-42b9-92db-c242f915eb40
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:50:06.315464Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15130"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822716477cc1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15138"
    uid: 99b0e809-3a8e-4c1c-92b1-a1cdbf3f7759
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15142"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: Add eth0 [10.133.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d682276da34685
    namespace: kserve-ci-e2e-test
    resourceVersion: "15169"
    uid: 654e56d7-6a1e-4c39-a31e-464f720bb81f
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d682276f01808f
    namespace: kserve-ci-e2e-test
    resourceVersion: "15171"
    uid: d888f6d2-40b1-4992-ac13-02b134bf6e60
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822772f66c5e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15172"
    uid: 262ca9b9-cf26-4047-9c20-b371504ccaf1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:07Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:50:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:07Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d68227736e220c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15173"
    uid: 6412581f-e9db-4b55-a3b5-9c22ad146a04
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:12Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
  metadata:
    creationTimestamp: "2026-09-18T19:50:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:12Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822875e2693b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15231"
    uid: 5474d2e6-1e07-4a8d-b9fe-8d475a0d29c8
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    in 16.025s (16.025s including waiting). Image size: 1583067017 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822c3114aa34
    namespace: kserve-ci-e2e-test
    resourceVersion: "15366"
    uid: ddd401b5-e46a-4ce5-bcd8-f0bda05b7173
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822c34a42b59
    namespace: kserve-ci-e2e-test
    resourceVersion: "15370"
    uid: 7c93c653-15cc-4e3e-863f-f23d358c3de2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822c3519ff83
    namespace: kserve-ci-e2e-test
    resourceVersion: "15371"
    uid: 92dd34e9-0461-434c-870a-da8b3ccb5dcf
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:28Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-18T19:50:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:28Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822c3526b366
    namespace: kserve-ci-e2e-test
    resourceVersion: "15372"
    uid: 189ef9df-6402-489a-ad13-9b69b4423ddc
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.594s (2.594s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822ccfc61a1d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15396"
    uid: 10748595-5de4-488c-8f1b-66d8e3a6f74d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:30Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:50:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:30Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822cd5225190
    namespace: kserve-ci-e2e-test
    resourceVersion: "15400"
    uid: 9508066d-3680-49b4-9baf-a9f72ecf303e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:31Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:50:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:31Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822cd5b5f1ba
    namespace: kserve-ci-e2e-test
    resourceVersion: "15402"
    uid: 1d1e781c-9f29-487d-aa37-af436b13f15c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:31Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:ee489751b997752a8855e389ed75068cf003229489532801b9b57f9452d3ed82"
  metadata:
    creationTimestamp: "2026-09-18T19:50:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:31Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822cd5c3de9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15403"
    uid: 6807bfec-b73c-4fe5-9d89-146ba62616b4
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:ee489751b997752a8855e389ed75068cf003229489532801b9b57f9452d3ed82"
    in 2.46s (2.46s including waiting). Image size: 243360834 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822d68668f9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15428"
    uid: 6ce6cbd7-ffd1-4077-83a9-78559843946b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822d6bc4bcda
    namespace: kserve-ci-e2e-test
    resourceVersion: "15429"
    uid: c1c42d73-ab58-4b91-845c-c66ef49bd63e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:50:33Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-18T19:50:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:50:33Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822d6c3c5bc0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15430"
    uid: 327e8571-60a3-459d-ad40-5c47b21a84a7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:52:16Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:50:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:16Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822d9e8d3b96
    namespace: kserve-ci-e2e-test
    resourceVersion: "16478"
    uid: 30124d55-05ff-43a1-8cd9-64cef206ca29
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:52:16Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-18T19:50:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:16Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d6822d9e956926
    namespace: kserve-ci-e2e-test
    resourceVersion: "16477"
    uid: aebc4a2b-62eb-4182-842f-b4b1da899cf5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:52:00Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:52:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:00Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d68241c7ba23f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16217"
    uid: 95cec0e2-e45c-4f97-84c2-087ab8cae0d8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:52:00Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-18T19:52:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:00Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d68241c7ba47ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "16218"
    uid: 2c1742b2-6783-48a7-badf-817612b64bae
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:52:00Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:52:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:00Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d68241c7baa135
    namespace: kserve-ci-e2e-test
    resourceVersion: "16221"
    uid: 2e61b520-2dfe-4557-a0e8-0c227e20981c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w
    namespace: kserve-ci-e2e-test
    resourceVersion: "15135"
    uid: 046aaf90-8702-48bc-888b-84e640067819
  kind: Event
  lastTimestamp: "2026-09-18T19:52:15Z"
  message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp
    10.133.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:52:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:15Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w.18d68242cec7bf4d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16456"
    uid: a32a5487-6ba7-41fe-907e-f6d857fce5a8
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15108"
    uid: 54d08855-7a00-490f-8916-06e4e58c151e
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: 'Created pod: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c-w4r5w'
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c.18d6822714df1ad4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15134"
    uid: 5f595293-519d-4fe3-9ecc-1c77509f364f
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-raw-sklearn-batcher-b7ffd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15106"
    uid: c2c5bc7a-6482-48bd-a963-5d4dae748672
  kind: Event
  lastTimestamp: "2026-09-18T19:50:06Z"
  message: Scaled up replica set isvc-raw-sklearn-batcher-b7ffd-predictor-588fd9b74c
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:50:06Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor.18d682270bc14206
    namespace: kserve-ci-e2e-test
    resourceVersion: "15109"
    uid: 46d2841e-ff77-47a0-ba0b-799f3672906d
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-b7ffd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15107"
    uid: 6d7ef370-54fd-4d9b-95df-370c25cbd29c
  kind: Event
  lastTimestamp: "2026-09-18T19:51:06Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:50:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:06Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor.18d6822a8b0ea7a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15689"
    uid: c16974ac-ad12-4f15-a184-0815d8fad41d
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-b7ffd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15107"
    uid: 6d7ef370-54fd-4d9b-95df-370c25cbd29c
  kind: Event
  lastTimestamp: "2026-09-18T19:51:06Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:50:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:06Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor.18d6822a8b74d7a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "15690"
    uid: 2db44a62-e8ed-4cd4-a13c-e115e7e184f3
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-b7ffd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15312"
    uid: 6d7ef370-54fd-4d9b-95df-370c25cbd29c
  kind: Event
  lastTimestamp: "2026-09-18T19:51:51Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:51:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:51Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor.18d68238877b1588
    namespace: kserve-ci-e2e-test
    resourceVersion: "16084"
    uid: 25525718-8744-4c94-9aca-c5f278726159
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-b7ffd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15312"
    uid: 6d7ef370-54fd-4d9b-95df-370c25cbd29c
  kind: Event
  lastTimestamp: "2026-09-18T19:51:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:51:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:51:51Z"
    name: isvc-raw-sklearn-batcher-b7ffd-predictor.18d6823887e938f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16085"
    uid: 96fbe12d-3989-44de-ac19-d3e6e41a90b2
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:50:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-b7ffd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15094"
    uid: 052a0e27-6127-4642-bf33-f6f1eb353789
  kind: Event
  lastTimestamp: "2026-09-18T19:51:46Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:50:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:51:46Z"
    name: isvc-raw-sklearn-batcher-b7ffd.18d682270bcefeae
    namespace: kserve-ci-e2e-test
    resourceVersion: "16025"
    uid: cb1da36b-0bcb-477d-a14a-6357c167751f
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:51:46Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-b7ffd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16023"
    uid: 052a0e27-6127-4642-bf33-f6f1eb353789
  kind: Event
  lastTimestamp: "2026-09-18T19:51:46Z"
  message: InferenceService [isvc-raw-sklearn-batcher-b7ffd] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:51:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:51:46Z"
    name: isvc-raw-sklearn-batcher-b7ffd.18d6823e69041d82
    namespace: kserve-ci-e2e-test
    resourceVersion: "16024"
    uid: cd74b049-bd4b-4c52-8e93-49e22e197c33
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:56:56.815009Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19434"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:56:56Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68286a9f58931
    namespace: kserve-ci-e2e-test
    resourceVersion: "19442"
    uid: 1eaafb10-486c-40d3-a778-25cb47e42755
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:56Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19438"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: "2026-09-18T19:56:56Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-81daee-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:56:56Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68286b4233783
    namespace: kserve-ci-e2e-test
    resourceVersion: "19450"
    uid: 473cf9f7-d956-4282-ad97-821b81c5c652
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:57Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19446"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: "2026-09-18T19:56:57Z"
  message: Add eth0 [10.134.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:56:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:56:57Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68286ef83013a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19458"
    uid: e8a1f92b-374c-40ee-a3ba-a7b7c41612c1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19438"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: "2026-09-18T19:57:02Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:56:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:02Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68286f1461bba
    namespace: kserve-ci-e2e-test
    resourceVersion: "19496"
    uid: d1d835bf-d961-4d9f-9246-dc62b62f592f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19438"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: "2026-09-18T19:57:02Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:56:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:02Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68286f40d1b0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19498"
    uid: 7ffb5087-bb39-49d2-805f-199e79494510
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19438"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: "2026-09-18T19:57:02Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:56:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:02Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68286f47f71cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "19499"
    uid: 55e24a20-6567-4983-901c-843ea7dabc3f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw
    namespace: kserve-ci-e2e-test
    resourceVersion: "19438"
    uid: 84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd
  kind: Event
  lastTimestamp: "2026-09-18T19:57:06Z"
  message: Back-off restarting failed container storage-initializer in pod isvc-secondary-81daee-predictor-5b59c775bf-rbtsw_kserve-ci-e2e-test(84e16c1d-53db-4f9e-bf30-f1e93d4ed9cd)
  metadata:
    creationTimestamp: "2026-09-18T19:57:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:06Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw.18d68288f28fe592
    namespace: kserve-ci-e2e-test
    resourceVersion: "19530"
    uid: 89b0a77a-b539-4f54-9e7d-354a77e9f157
  reason: BackOff
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:56Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-secondary-81daee-predictor-5b59c775bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "19426"
    uid: 192b1aad-a349-4bb8-94a7-29bd35c6a5ca
  kind: Event
  lastTimestamp: "2026-09-18T19:56:56Z"
  message: 'Created pod: isvc-secondary-81daee-predictor-5b59c775bf-rbtsw'
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:56Z"
    name: isvc-secondary-81daee-predictor-5b59c775bf.18d68286a97dac12
    namespace: kserve-ci-e2e-test
    resourceVersion: "19437"
    uid: a682f444-5f84-4785-a546-41fafe90dd3f
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:56Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-secondary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19424"
    uid: 0522a75c-789b-4ef5-8baa-f7e6c5db7622
  kind: Event
  lastTimestamp: "2026-09-18T19:56:56Z"
  message: Scaled up replica set isvc-secondary-81daee-predictor-5b59c775bf from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:56Z"
    name: isvc-secondary-81daee-predictor.18d68286a5eaa879
    namespace: kserve-ci-e2e-test
    resourceVersion: "19428"
    uid: b01b22b7-96e2-4062-8834-4064bdb92118
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19425"
    uid: 2b0cb41a-e80a-4c5c-8720-2bb19b54ac8d
  kind: Event
  lastTimestamp: "2026-09-18T19:57:11Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:57:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:11Z"
    name: isvc-secondary-81daee-predictor.18d6828a2474b50a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19572"
    uid: c2959a18-10a3-475e-9832-f7a894d17f4a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-81daee-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19425"
    uid: 2b0cb41a-e80a-4c5c-8720-2bb19b54ac8d
  kind: Event
  lastTimestamp: "2026-09-18T19:57:11Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:57:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:11Z"
    name: isvc-secondary-81daee-predictor.18d6828a24ddbf78
    namespace: kserve-ci-e2e-test
    resourceVersion: "19574"
    uid: 8c877342-31af-48f2-bdeb-36d79d50c590
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:56Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "19417"
    uid: 53bdc30c-82af-4ea2-a485-abcfc726b1d9
  kind: Event
  lastTimestamp: "2026-09-18T19:57:06Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:06Z"
    name: isvc-secondary-81daee.18d68286a5d4c8e1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19539"
    uid: 7d707047-6845-4aae-b2fb-927442d53e3d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:56Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "19417"
    uid: 53bdc30c-82af-4ea2-a485-abcfc726b1d9
  kind: Event
  lastTimestamp: "2026-09-18T19:56:56Z"
  message: 'Failed to update status for InferenceService "isvc-secondary-81daee":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-81daee":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:56:56Z"
    name: isvc-secondary-81daee.18d68286a6bc5aeb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19430"
    uid: c67d9ce0-66b5-460d-96a2-32f5314d1bd5
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:56Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-81daee
    namespace: kserve-ci-e2e-test
    resourceVersion: "19417"
    uid: 53bdc30c-82af-4ea2-a485-abcfc726b1d9
  kind: Event
  lastTimestamp: "2026-09-18T19:56:56Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-secondary-81daee": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:56:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:56:56Z"
    name: isvc-secondary-81daee.18d68286a6bcbd51
    namespace: kserve-ci-e2e-test
    resourceVersion: "19431"
    uid: 63dc18be-5eaa-4748-aeac-a5ff73fd352f
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:52:01.112202Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16247"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d68241d0b2afe1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16252"
    uid: 4eb81357-41ff-48ba-a77e-85541df3bc90
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16261"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Add eth0 [10.134.0.20/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d68241f313c047
    namespace: kserve-ci-e2e-test
    resourceVersion: "16310"
    uid: 844084de-cfd5-4ff8-b84f-e9f00e06edc4
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d68241f47739c9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16312"
    uid: 08697a5c-41a8-4ed0-b6a7-4166207dd530
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d68241f7036378
    namespace: kserve-ci-e2e-test
    resourceVersion: "16313"
    uid: 646ff10b-4fe0-45c8-bf61-fba7408e7df9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d68241f7775c7c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16315"
    uid: 7b4868ea-e61b-4fda-aaaf-10728af2d665
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d682430ce331aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "16370"
    uid: 95ddabbf-6e23-4d38-bf76-b7c7056a2d87
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d6824310f800a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16376"
    uid: 4b25e454-988e-4d74-8416-49ebfa1e0bba
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d68243116929f5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16377"
    uid: fdda40bd-4257-4c7e-908a-cdd65386fae0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d682431174bd10
    namespace: kserve-ci-e2e-test
    resourceVersion: "16378"
    uid: d2a324d0-5d5f-4b3c-b5a8-075d20111a37
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d6824314332285
    namespace: kserve-ci-e2e-test
    resourceVersion: "16381"
    uid: 7e4f832e-2fd2-4083-8690-5e79f5a70c2b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d6824314a16db8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16382"
    uid: 8296687a-6228-47ea-91cd-5b881c1330cc
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:53:53Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:52:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:53Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d682434925104c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17574"
    uid: b0ddb9bf-7952-4e55-b556-a44dafbc6410
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d6825b79932244
    namespace: kserve-ci-e2e-test
    resourceVersion: "17446"
    uid: bf6bcd83-a5bb-4918-a6e9-ded8bbe4cdc5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d6825b799383a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17448"
    uid: 66aeb884-f883-413e-bac5-77e8e614303d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:53Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9
    namespace: kserve-ci-e2e-test
    resourceVersion: "16250"
    uid: 292307d8-3fb5-4b4a-9531-fa2f622588c4
  kind: Event
  lastTimestamp: "2026-09-18T19:53:53Z"
  message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp
    10.134.0.20:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:53Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:53Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9.18d6825bf77a9a3e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17573"
    uid: c2e935e1-8d5f-4098-9a6a-df9c2ec6a864
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16236"
    uid: 96f52891-10a6-4352-bbbf-d7e0fd50170d
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'Created pod: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c-krst9'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor-cbd89768c.18d68241d0203ffe
    namespace: kserve-ci-e2e-test
    resourceVersion: "16249"
    uid: fb8e8362-206e-45ab-abc9-94265e51db7d
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16234"
    uid: 7e8c2835-bd9b-4b8b-93f8-034124e2ba35
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-2f453-predictor-cbd89768c
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453-predictor.18d68241cd31c7a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16238"
    uid: e3ebc48b-3fc6-4ac6-a07f-eac7159f8d67
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16235"
    uid: b97b4c82-ed52-416e-9890-8c3e909a08f2
  kind: Event
  lastTimestamp: "2026-09-18T19:52:16Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:52:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:16Z"
    name: isvc-sklearn-graph-raw-2f453-predictor.18d682454bb548a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16461"
    uid: d07b9db2-71d7-469e-8e7c-040a8a218d91
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16235"
    uid: b97b4c82-ed52-416e-9890-8c3e909a08f2
  kind: Event
  lastTimestamp: "2026-09-18T19:52:16Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:52:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:16Z"
    name: isvc-sklearn-graph-raw-2f453-predictor.18d682454c172a61
    namespace: kserve-ci-e2e-test
    resourceVersion: "16463"
    uid: 3c64fef2-ee85-4a5c-98e9-78eb612aa8ed
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:31Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16462"
    uid: b97b4c82-ed52-416e-9890-8c3e909a08f2
  kind: Event
  lastTimestamp: "2026-09-18T19:53:46Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:52:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:46Z"
    name: isvc-sklearn-graph-raw-2f453-predictor.18d68248cb23a60d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17364"
    uid: d9b8f45e-f9be-43cb-9d4b-89021ec74abb
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:31Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16462"
    uid: b97b4c82-ed52-416e-9890-8c3e909a08f2
  kind: Event
  lastTimestamp: "2026-09-18T19:53:46Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:52:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:46Z"
    name: isvc-sklearn-graph-raw-2f453-predictor.18d68248cb8e11f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17366"
    uid: 9d2a0ef8-ec5f-497d-bb15-64098c79295a
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "16225"
    uid: 5689b1a3-69a7-4721-94a0-3fbb2afb30aa
  kind: Event
  lastTimestamp: "2026-09-18T19:53:23Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:53:23Z"
    name: isvc-sklearn-graph-raw-2f453.18d68241cd68c372
    namespace: kserve-ci-e2e-test
    resourceVersion: "16997"
    uid: c4acfe47-f6a0-4397-b959-0f81dc393e51
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "16225"
    uid: 5689b1a3-69a7-4721-94a0-3fbb2afb30aa
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-2f453":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-2f453":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-sklearn-graph-raw-2f453.18d68241cdd5db19
    namespace: kserve-ci-e2e-test
    resourceVersion: "16242"
    uid: 8331e43d-e4a7-4fcf-b651-8214392f457c
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "16994"
    uid: 5689b1a3-69a7-4721-94a0-3fbb2afb30aa
  kind: Event
  lastTimestamp: "2026-09-18T19:53:23Z"
  message: InferenceService [isvc-sklearn-graph-raw-2f453] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:53:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:53:23Z"
    name: isvc-sklearn-graph-raw-2f453.18d682550115f5a8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16995"
    uid: e73b7a36-2fd6-4bce-8e27-8b548f755a1a
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:53:51.464361Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17487"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825b82332f37
    namespace: kserve-ci-e2e-test
    resourceVersion: "17491"
    uid: e96453be-862f-4e52-8212-308199f146c1
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17500"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: Add eth0 [10.133.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ba4863a58
    namespace: kserve-ci-e2e-test
    resourceVersion: "17539"
    uid: cc3d674e-7be5-4c81-874c-5a6e836259a1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ba5bfc9d5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17541"
    uid: 94564e0e-523a-4111-93b5-c824a87d711b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ba876b8da
    namespace: kserve-ci-e2e-test
    resourceVersion: "17542"
    uid: 68129cf2-92f5-41ce-b6d1-25eaefd87b98
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ba8d5b319
    namespace: kserve-ci-e2e-test
    resourceVersion: "17543"
    uid: e01da2db-438e-407e-b05f-2298d7eaffbd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:56Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ca46181ac
    namespace: kserve-ci-e2e-test
    resourceVersion: "17602"
    uid: d4ae59e5-f751-4b91-86f9-ed28e45e7e64
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:56Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ca7532685
    namespace: kserve-ci-e2e-test
    resourceVersion: "17604"
    uid: 7887113a-6279-4787-bd36-c326f03d07bf
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:56Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:56Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ca7b4e516
    namespace: kserve-ci-e2e-test
    resourceVersion: "17606"
    uid: 22928037-3419-4f9a-93da-b6ec99ff3a87
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:56Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:56Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825ca7bdd3fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17608"
    uid: 3803d6d7-a945-4c7d-b1e2-4326d559b882
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:56Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:56Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825caae2a707
    namespace: kserve-ci-e2e-test
    resourceVersion: "17610"
    uid: 3149dc19-eb40-4ae6-9c82-b9aaa41afb86
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:53:56Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:56Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825cab4777f2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17611"
    uid: e3d9cef8-6fa2-4517-a2e4-c3cc9ccabe6b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d6825d1c9f69dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18763"
    uid: d2639243-efde-4d46-b540-c4e6d3691c9c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d68275292d1a70
    namespace: kserve-ci-e2e-test
    resourceVersion: "18675"
    uid: 0db2e73c-3956-47f1-aa1b-a9f3f970f814
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d68275292dd23c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18676"
    uid: 77d0dad1-bf42-4751-bfa4-2f9d7ce878ee
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: f0bcc744-7e00-4299-950c-59018f18ecc6
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp
    10.133.0.30:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf.18d68275cb14a245
    namespace: kserve-ci-e2e-test
    resourceVersion: "18762"
    uid: 886d946b-4b28-4e19-9176-6cea0c13bcc2
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955
    namespace: kserve-ci-e2e-test
    resourceVersion: "17477"
    uid: d01ecfdd-a946-4aec-a414-cb7e52189a2a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: 'Created pod: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955-jtnrf'
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955.18d6825b81c7cc08
    namespace: kserve-ci-e2e-test
    resourceVersion: "17488"
    uid: fb364d41-9f1d-48bf-94bf-d492178367e4
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17474"
    uid: 0f8039ac-1841-4416-952a-313363094c61
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-hpa-e0026-predictor-5f4bf59955
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor.18d6825b7f3f44c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17478"
    uid: 2b0f03b4-a0ad-4746-9976-44a2e9ee5573
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17475"
    uid: 46a2aae1-6b09-493b-a663-2c68b0bfa63b
  kind: Event
  lastTimestamp: "2026-09-18T19:54:21Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:54:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:54:21Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor.18d6825efdc1ce32
    namespace: kserve-ci-e2e-test
    resourceVersion: "17824"
    uid: 59db8c84-380a-4009-8e31-27a48afdc2ac
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17475"
    uid: 46a2aae1-6b09-493b-a663-2c68b0bfa63b
  kind: Event
  lastTimestamp: "2026-09-18T19:54:21Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:54:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:54:21Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor.18d6825efe2e54d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17825"
    uid: 3bf3d89d-be99-4c78-bc2b-aa29aaa23a1b
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17702"
    uid: 46a2aae1-6b09-493b-a663-2c68b0bfa63b
  kind: Event
  lastTimestamp: "2026-09-18T19:55:36Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:54:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:36Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor.18d68265fc1d3c60
    namespace: kserve-ci-e2e-test
    resourceVersion: "18596"
    uid: d6d8ddb9-50ce-49d8-b3e6-f15e5558a738
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:36Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17702"
    uid: 46a2aae1-6b09-493b-a663-2c68b0bfa63b
  kind: Event
  lastTimestamp: "2026-09-18T19:55:36Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:54:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:36Z"
    name: isvc-sklearn-graph-raw-hpa-e0026-predictor.18d68265fc8839f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "18597"
    uid: d9bf5285-9c90-4b76-9f5a-90100614b378
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-e0026
    namespace: kserve-ci-e2e-test
    resourceVersion: "17464"
    uid: a5f5a461-acf8-43f1-b316-8a84f7c53977
  kind: Event
  lastTimestamp: "2026-09-18T19:55:14Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:14Z"
    name: isvc-sklearn-graph-raw-hpa-e0026.18d6825b7f46c244
    namespace: kserve-ci-e2e-test
    resourceVersion: "18380"
    uid: 55e48673-668b-4672-ba89-9bf94d567248
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:14Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-e0026
    namespace: kserve-ci-e2e-test
    resourceVersion: "18378"
    uid: a5f5a461-acf8-43f1-b316-8a84f7c53977
  kind: Event
  lastTimestamp: "2026-09-18T19:55:14Z"
  message: InferenceService [isvc-sklearn-graph-raw-hpa-e0026] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:55:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:14Z"
    name: isvc-sklearn-graph-raw-hpa-e0026.18d6826ed4760ae2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18379"
    uid: c4af0b64-ab09-4352-b887-8d0b7c35ca9c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:57:27.070265Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19849"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:57:27Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:57:27Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828db5501288
    namespace: kserve-ci-e2e-test
    resourceVersion: "19855"
    uid: b07839d7-b992-416c-9c14-7bafd7ad79af
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:27Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:27Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-1efdd-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:57:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:27Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828dc4f5fb8a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19872"
    uid: 773267ad-c4af-4edc-a3f9-e2f131cdbd6d
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:28Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19863"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:28Z"
  message: Add eth0 [10.134.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:57:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:57:28Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828dfe7dbf1b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19882"
    uid: 7e3a8f69-947a-450b-82e5-d2d2d182f961
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:28Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:28Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828dffb4649f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19884"
    uid: c6a21f86-0645-443f-b323-e4a20dbfdcef
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:28Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:57:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:28Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828e02963816
    namespace: kserve-ci-e2e-test
    resourceVersion: "19885"
    uid: 84cb1e5d-85d2-4499-b3e4-f2c41661708f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:28Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:57:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:28Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828e030e4f34
    namespace: kserve-ci-e2e-test
    resourceVersion: "19886"
    uid: 348d28fc-cf21-4934-976b-3738627ebc50
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:32Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:32Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f06cae409
    namespace: kserve-ci-e2e-test
    resourceVersion: "19948"
    uid: bdf4803b-f4f7-4af0-9823-e3456b6bfe5d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:32Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:57:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:32Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f09b15f2d
    namespace: kserve-ci-e2e-test
    resourceVersion: "19950"
    uid: af1a4203-63d9-47c6-8569-eed3e91cdfd1
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:32Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:57:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:32Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f0a22cdff
    namespace: kserve-ci-e2e-test
    resourceVersion: "19953"
    uid: c42741c9-4420-48c1-8e8f-eed05da4ba32
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:32Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:32Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f0a2bc901
    namespace: kserve-ci-e2e-test
    resourceVersion: "19955"
    uid: 723cbae6-2e3e-4083-83e5-844238bef4b1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:32Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:57:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:32Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f0cc08d44
    namespace: kserve-ci-e2e-test
    resourceVersion: "19958"
    uid: 4e1a10d7-2252-4337-871e-a700f43dbcda
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T19:57:32Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:57:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:32Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f0d32a214
    namespace: kserve-ci-e2e-test
    resourceVersion: "19959"
    uid: cb5ace2c-77c5-417b-b420-390cd497c8ca
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T20:00:12Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:57:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:12Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d6828f42e1a589
    namespace: kserve-ci-e2e-test
    resourceVersion: "21379"
    uid: d3e11eb2-87f1-4b48-91e9-6204aa86af55
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T20:00:07Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:00:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:07Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d682b2f7dc3797
    namespace: kserve-ci-e2e-test
    resourceVersion: "21282"
    uid: 432debc8-a89c-40c2-9ff7-0b10a812e8ea
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T20:00:07Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:00:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:07Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d682b2f7dcd6e2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21283"
    uid: 12abfdd6-dbac-4b6b-a3bb-577b0f9dfb43
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p
    namespace: kserve-ci-e2e-test
    resourceVersion: "19853"
    uid: ba6a0aff-e782-488a-8949-474b9c442f33
  kind: Event
  lastTimestamp: "2026-09-18T20:00:15Z"
  message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp
    10.134.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:00:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:15Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p.18d682b3d1319a4c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21443"
    uid: d6927232-3e06-4c82-87cf-b84665e7aa90
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:27Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65
    namespace: kserve-ci-e2e-test
    resourceVersion: "19828"
    uid: 8a9b891b-0b3f-42f0-b0b1-8ad0f01b878b
  kind: Event
  lastTimestamp: "2026-09-18T19:57:27Z"
  message: 'Created pod: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65-4sq8p'
  metadata:
    creationTimestamp: "2026-09-18T19:57:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:27Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65.18d6828db495c83b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19851"
    uid: 42f439ae-8404-4bcf-b14f-d39c26f6c2a1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:26Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-scale-raw-1efdd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19826"
    uid: a183c859-3105-488f-b5db-63503ddeaaaf
  kind: Event
  lastTimestamp: "2026-09-18T19:57:26Z"
  message: Scaled up replica set isvc-sklearn-scale-raw-1efdd-predictor-6f5d857b65
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:26Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor.18d6828daff102fd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19830"
    uid: 2a8189aa-6425-4488-bd4d-6ce56b2bc013
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:41Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-1efdd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19827"
    uid: afa0ce8a-536d-4b78-8f51-58d77012a837
  kind: Event
  lastTimestamp: "2026-09-18T19:58:12Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:57:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:58:12Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor.18d682912e63cd5b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20328"
    uid: 51b5e932-68a7-4e17-8ad8-ef899d8c3b3f
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:41Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-1efdd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19827"
    uid: afa0ce8a-536d-4b78-8f51-58d77012a837
  kind: Event
  lastTimestamp: "2026-09-18T19:58:12Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:57:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:58:12Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor.18d682912ec94de8
    namespace: kserve-ci-e2e-test
    resourceVersion: "20329"
    uid: 38b0535a-2d8a-4492-b692-cfe529cac4bb
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-09-18T19:58:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-1efdd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20092"
    uid: afa0ce8a-536d-4b78-8f51-58d77012a837
  kind: Event
  lastTimestamp: "2026-09-18T19:59:57Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:58:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:57Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor.18d6829bab73580a
    namespace: kserve-ci-e2e-test
    resourceVersion: "21197"
    uid: be7a0202-2998-495e-a957-fc8ef6b55608
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-09-18T19:58:27Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-1efdd-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20092"
    uid: afa0ce8a-536d-4b78-8f51-58d77012a837
  kind: Event
  lastTimestamp: "2026-09-18T19:59:57Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:58:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:57Z"
    name: isvc-sklearn-scale-raw-1efdd-predictor.18d6829babd441b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21198"
    uid: 3c7efed1-b1ef-455c-ac94-7746750cc324
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:26Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-1efdd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19815"
    uid: 172a3932-cfa5-47f9-ad33-d7e227293282
  kind: Event
  lastTimestamp: "2026-09-18T20:00:02Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:57:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:00:02Z"
    name: isvc-sklearn-scale-raw-1efdd.18d6828db0202bb7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21240"
    uid: 3650436e-88b1-4699-8584-4544ac5080ba
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-1efdd
    namespace: kserve-ci-e2e-test
    resourceVersion: "21237"
    uid: 172a3932-cfa5-47f9-ad33-d7e227293282
  kind: Event
  lastTimestamp: "2026-09-18T20:00:02Z"
  message: InferenceService [isvc-sklearn-scale-raw-1efdd] is Ready
  metadata:
    creationTimestamp: "2026-09-18T20:00:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:00:02Z"
    name: isvc-sklearn-scale-raw-1efdd.18d682b1e8835f7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21238"
    uid: 802c4a41-31f7-433b-bec8-2e54b85699d1
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:52:01.209030Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16273"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d68241d6783c8e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16277"
    uid: c501f9b6-5186-4383-9f8a-6d13c18bbfc4
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-2f453-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d68241e21c9894
    namespace: kserve-ci-e2e-test
    resourceVersion: "16294"
    uid: a6b263af-f9b7-47e0-9d57-457b33e3acda
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16283"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:02Z"
  message: Add eth0 [10.134.0.21/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:52:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:52:02Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682421c0f6bf1
    namespace: kserve-ci-e2e-test
    resourceVersion: "16318"
    uid: 1ae6a15e-f23c-462e-9db1-5d48003553b1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:02Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:02Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682421d722066
    namespace: kserve-ci-e2e-test
    resourceVersion: "16321"
    uid: 49376cbd-8519-4b3e-bed0-d829c0fb5179
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:02Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:52:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:02Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d6824220a0db03
    namespace: kserve-ci-e2e-test
    resourceVersion: "16326"
    uid: e84ab9e9-533d-485e-b12e-23524c31d9d0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:02Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:52:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:02Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d68242211459d5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16327"
    uid: 72b1b87c-3d74-4b24-ab45-604e0f4f4a5e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:06Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-09-18T19:52:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:06Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682430d00b8a2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16371"
    uid: 6285ff55-d66d-4dbb-aecf-b12b43add3d0
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:25Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.928s (18.928s
    including waiting). Image size: 1436472579 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:25Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d68247753c8e32
    namespace: kserve-ci-e2e-test
    resourceVersion: "16537"
    uid: 4741a510-fabc-456b-8357-bcb2ea4bfe9c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:25Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:25Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682477b688d79
    namespace: kserve-ci-e2e-test
    resourceVersion: "16540"
    uid: 0353b18b-e939-4079-83d3-6fe37cbe5fb9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:25Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:25Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682477c7d7dec
    namespace: kserve-ci-e2e-test
    resourceVersion: "16541"
    uid: 38e95b16-b16f-40c4-bf01-47f0a265b672
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:25Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:25Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682477cc9601f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16542"
    uid: 9a6960d8-b330-476b-b090-df26c8876881
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:25Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:25Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d682478315382c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16543"
    uid: bf397d04-e62b-403e-b034-1da6033f8e4c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:52:25Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:52:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:52:25Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d68247842460d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16545"
    uid: 82534096-628c-4691-a18c-5595a41300fd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:52:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d68247bad7a0f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17556"
    uid: 092a6764-9524-47d7-9f1f-7da54aca6ea9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d6825b7ba27802
    namespace: kserve-ci-e2e-test
    resourceVersion: "17465"
    uid: 4d1927fd-a9c4-41d1-a2d8-8df3c6d22e52
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d6825b7ba2f0d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17467"
    uid: 4ebbb47a-8cff-4a2d-aa0c-24bbcad2cb1b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx
    namespace: kserve-ci-e2e-test
    resourceVersion: "16275"
    uid: a564cac9-6b13-4726-8f8f-ebd8f7af0b57
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp
    10.134.0.21:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx.18d6825bc1036047
    namespace: kserve-ci-e2e-test
    resourceVersion: "17555"
    uid: 17f284cf-ef82-4d33-8637-ed5f86ec8a7f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df
    namespace: kserve-ci-e2e-test
    resourceVersion: "16265"
    uid: ca68b26c-5ecf-4d51-a937-fe67a5e78e9d
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: 'Created pod: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df-5qhrx'
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df.18d68241d60e5b46
    namespace: kserve-ci-e2e-test
    resourceVersion: "16274"
    uid: 9492b0df-213a-459c-931e-8258dcc05a00
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16262"
    uid: 83dc1c2a-4620-4aec-a034-f1896ad65df3
  kind: Event
  lastTimestamp: "2026-09-18T19:52:01Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-2f453-predictor-6f4b8856df
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:01Z"
    name: isvc-xgboost-graph-raw-2f453-predictor.18d68241d35074b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16267"
    uid: 9b4d2e12-969b-4468-88a3-4ee34b9b0204
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16264"
    uid: a5ea69a5-937a-40a1-a5a4-6b669034d2ba
  kind: Event
  lastTimestamp: "2026-09-18T19:52:46Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:52:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:46Z"
    name: isvc-xgboost-graph-raw-2f453-predictor.18d6824551d80d0b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16703"
    uid: 38518a8f-5bab-472f-a1d9-35fbd29a3d29
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16264"
    uid: a5ea69a5-937a-40a1-a5a4-6b669034d2ba
  kind: Event
  lastTimestamp: "2026-09-18T19:52:46Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:52:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:52:46Z"
    name: isvc-xgboost-graph-raw-2f453-predictor.18d682455234e3a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16704"
    uid: aa2c4895-57a5-4ae0-9f82-673ff2b4d46f
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:01Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16467"
    uid: a5ea69a5-937a-40a1-a5a4-6b669034d2ba
  kind: Event
  lastTimestamp: "2026-09-18T19:53:46Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:53:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:46Z"
    name: isvc-xgboost-graph-raw-2f453-predictor.18d6824fce3b5091
    namespace: kserve-ci-e2e-test
    resourceVersion: "17367"
    uid: d52bc067-f1dc-408c-ab36-1442f276e875
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:01Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-2f453-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16467"
    uid: a5ea69a5-937a-40a1-a5a4-6b669034d2ba
  kind: Event
  lastTimestamp: "2026-09-18T19:53:46Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:53:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:46Z"
    name: isvc-xgboost-graph-raw-2f453-predictor.18d6824fce9afd0f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17368"
    uid: 7402fc30-be7d-4054-a8f0-ed9be57fc527
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:52:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "16246"
    uid: a9a199c2-e338-4f6d-bc4b-57812834311a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:52:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-xgboost-graph-raw-2f453.18d68241d37f993e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17183"
    uid: 4807cdac-2c28-444b-a9fb-2c57c4cef9e8
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:32Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "17179"
    uid: a9a199c2-e338-4f6d-bc4b-57812834311a
  kind: Event
  lastTimestamp: "2026-09-18T19:53:32Z"
  message: InferenceService [isvc-xgboost-graph-raw-2f453] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:53:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:53:32Z"
    name: isvc-xgboost-graph-raw-2f453.18d682571eec8d86
    namespace: kserve-ci-e2e-test
    resourceVersion: "17180"
    uid: bd61bc87-111b-4d7f-a8ca-82635436221c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:53:51.569421Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17515"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    to ip-10-0-143-198.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825b88762cba
    namespace: kserve-ci-e2e-test
    resourceVersion: "17519"
    uid: be712909-e2af-499d-a126-ea17be32496b
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-e0026-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825b971de632
    namespace: kserve-ci-e2e-test
    resourceVersion: "17532"
    uid: cdcea2ec-f65d-47c3-a638-936b75a1c9d7
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17525"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: Add eth0 [10.133.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825bce1aceb6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17558"
    uid: 4f3b589b-bc1b-450c-ab9a-36c174f1e782
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825bcf7bbdec
    namespace: kserve-ci-e2e-test
    resourceVersion: "17560"
    uid: 0314551b-b7d5-44b3-8737-2f2c42d9d35d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825bd3041efd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17561"
    uid: cece255d-acff-430c-9355-5ce3a2056b98
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:53:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:53:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:52Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825bd3680c3b
    namespace: kserve-ci-e2e-test
    resourceVersion: "17562"
    uid: 97dafdc3-c114-4260-bf39-2d9e13c00cbb
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:53:57Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-09-18T19:53:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:57Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6825ce06b300f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17618"
    uid: e32b512c-db62-4fa1-85f2-72a41d8d7c4e
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:54:16Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.575s (19.575s
    including waiting). Image size: 1436472579 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:54:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:16Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d682616f39663d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17783"
    uid: e591af9d-b8ea-446e-92cf-eeb0e384bd15
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:54:16Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:54:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:16Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d68261725c0bfe
    namespace: kserve-ci-e2e-test
    resourceVersion: "17785"
    uid: f0d23997-cfb4-415e-a3eb-895fbab5b5f2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:54:16Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:54:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:16Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6826172cd1dd4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17786"
    uid: 21170957-44ff-4675-a4cc-26598e67f8cd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:54:16Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:54:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:16Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6826172d75ad9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17787"
    uid: 214dc0f2-8c04-4f2f-994b-df8f0af1caeb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:54:17Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:54:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:17Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d68261759c1dda
    namespace: kserve-ci-e2e-test
    resourceVersion: "17788"
    uid: c6783333-b6f9-4e3b-b295-d1d34ee0762c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:54:17Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:54:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:17Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d68261760cf2ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "17789"
    uid: 03f84008-248a-4b6c-9bd9-01b79fce88d5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:55:43Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:54:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:43Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d682618f62acd9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18746"
    uid: 0b2e7bd5-6773-4f86-8058-8b64ccb348ba
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6827531aa88ce
    namespace: kserve-ci-e2e-test
    resourceVersion: "18718"
    uid: bd1b323e-027c-4609-90f3-bf79f64fecd8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d6827531aaf909
    namespace: kserve-ci-e2e-test
    resourceVersion: "18721"
    uid: 9391d093-1145-427e-9b1d-5f8e6f91e604
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl
    namespace: kserve-ci-e2e-test
    resourceVersion: "17517"
    uid: faf2e9f7-e654-4e5b-89e1-ddb348f7d0bd
  kind: Event
  lastTimestamp: "2026-09-18T19:55:43Z"
  message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp
    10.133.0.31:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:55:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:43Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl.18d682759596b9f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18745"
    uid: b8de9f88-13f4-4b55-9576-a2757917aff9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-198.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-198.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df
    namespace: kserve-ci-e2e-test
    resourceVersion: "17504"
    uid: 8a595169-4d13-4341-9b56-576efbd031b7
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: 'Created pod: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df-cq4rl'
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df.18d6825b8809a6ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "17516"
    uid: e7537d72-2266-4a8f-a516-3b796fb5edca
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17501"
    uid: 8c906eeb-db3c-46f0-8866-bb05971e5a31
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-hpa-e0026-predictor-85549564df
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor.18d6825b852a5d7d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17506"
    uid: b6c0f8c0-1d0e-4846-8215-fc10efaa1d19
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17503"
    uid: d47b94e7-803b-417c-916c-4a2c55cafcc4
  kind: Event
  lastTimestamp: "2026-09-18T19:54:51Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:54:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:54:51Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor.18d6825f039b2472
    namespace: kserve-ci-e2e-test
    resourceVersion: "18056"
    uid: ac02e218-4a43-4ffd-99ad-2b837da96305
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:54:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17503"
    uid: d47b94e7-803b-417c-916c-4a2c55cafcc4
  kind: Event
  lastTimestamp: "2026-09-18T19:54:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:54:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:54:51Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor.18d6825f03fb20d8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18057"
    uid: 9399a37b-b1d4-45f3-84d9-f9935b210798
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17704"
    uid: d47b94e7-803b-417c-916c-4a2c55cafcc4
  kind: Event
  lastTimestamp: "2026-09-18T19:55:36Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:55:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:36Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor.18d6826cffdc7088
    namespace: kserve-ci-e2e-test
    resourceVersion: "18599"
    uid: 1d2355a5-922e-4bd2-9c4c-b433e24a6b08
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17704"
    uid: d47b94e7-803b-417c-916c-4a2c55cafcc4
  kind: Event
  lastTimestamp: "2026-09-18T19:55:36Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:55:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:36Z"
    name: isvc-xgboost-graph-raw-hpa-e0026-predictor.18d6826d00a1fbcb
    namespace: kserve-ci-e2e-test
    resourceVersion: "18600"
    uid: 8d438c1c-e785-4cbf-b370-be4ebe9b3d3b
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-e0026
    namespace: kserve-ci-e2e-test
    resourceVersion: "17486"
    uid: ed622efa-aea1-422f-abb6-7abfab2101d3
  kind: Event
  lastTimestamp: "2026-09-18T19:55:23Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:23Z"
    name: isvc-xgboost-graph-raw-hpa-e0026.18d6825b852bbe72
    namespace: kserve-ci-e2e-test
    resourceVersion: "18461"
    uid: 3eef91e9-9e67-479f-9388-2dc9eb25b05e
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-e0026
    namespace: kserve-ci-e2e-test
    resourceVersion: "18459"
    uid: ed622efa-aea1-422f-abb6-7abfab2101d3
  kind: Event
  lastTimestamp: "2026-09-18T19:55:23Z"
  message: InferenceService [isvc-xgboost-graph-raw-hpa-e0026] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:55:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:23Z"
    name: isvc-xgboost-graph-raw-hpa-e0026.18d68270f530096b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18460"
    uid: 779c794d-539d-4df6-8b59-fb6de434591b
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:55:41.725651Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18694"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d682752e48d465
    namespace: kserve-ci-e2e-test
    resourceVersion: "18698"
    uid: 267aa8ff-740c-4bdc-bf10-516990b1bd0a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-91b5b-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d682753a2f1bbc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18725"
    uid: b6a822c7-a973-4c91-8359-6a20120eaa54
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:42Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18711"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:42Z"
  message: Add eth0 [10.134.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:55:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:55:42Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d6827574030993
    namespace: kserve-ci-e2e-test
    resourceVersion: "18737"
    uid: 73b4ba27-e87e-495a-ad65-7ba71767bf10
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:42Z"
  message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
  metadata:
    creationTimestamp: "2026-09-18T19:55:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:42Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d682757556c621
    namespace: kserve-ci-e2e-test
    resourceVersion: "18739"
    uid: 17401332-252e-499c-b8ec-38e9386a8d44
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    in 1.087s (1.087s including waiting). Image size: 14813193 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d68275b62ac563
    namespace: kserve-ci-e2e-test
    resourceVersion: "18749"
    uid: fddbb501-199b-4f6d-bf84-b374cca87113
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d68275b8fbb586
    namespace: kserve-ci-e2e-test
    resourceVersion: "18750"
    uid: 9b11edf7-7d69-4726-b010-bc91534bcffe
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d68275b9706920
    namespace: kserve-ci-e2e-test
    resourceVersion: "18751"
    uid: 90f04b01-418c-4423-9de8-776cb3af269f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d68275b97bacc7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18752"
    uid: ff1bd6e7-63c5-4e1a-9a7a-e27d9d68225b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d68275bd2ae908
    namespace: kserve-ci-e2e-test
    resourceVersion: "18754"
    uid: 9d9e11fc-7ccf-419f-9434-3c899e3fe363
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:44Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:55:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:44Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d68275bda2f52b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18755"
    uid: 84ac88f4-f06b-43de-a42f-e6e05e004368
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:27Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:57:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:27Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d6828db4f56d9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "19854"
    uid: afffaa5a-bc4d-4045-ab95-d3620e07614e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89
    namespace: kserve-ci-e2e-test
    resourceVersion: "18696"
    uid: 192fe0a2-3850-4a97-bbf4-1c374a74db3c
  kind: Event
  lastTimestamp: "2026-09-18T19:57:27Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:57:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:27Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89.18d6828db4f5e917
    namespace: kserve-ci-e2e-test
    resourceVersion: "19856"
    uid: 118e47ad-cb8d-48ec-b68d-116e482c763b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-91b5b-predictor-598d8989f5
    namespace: kserve-ci-e2e-test
    resourceVersion: "18686"
    uid: fffd863a-1999-44cc-a09d-bc62c29aea56
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: 'Created pod: message-dumper-raw-91b5b-predictor-598d8989f5-8jv89'
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: message-dumper-raw-91b5b-predictor-598d8989f5.18d682752dd8434b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18695"
    uid: 52d8a033-1e0c-4e13-b535-524a8a22b46b
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18683"
    uid: 6e36eef5-c123-4ae9-b7ac-05820ff2a84b
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: Scaled up replica set message-dumper-raw-91b5b-predictor-598d8989f5 from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: message-dumper-raw-91b5b-predictor.18d682752beb237a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18687"
    uid: 56d7df58-940a-438c-ad96-92df483bae32
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:56Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18684"
    uid: ff517e4b-5b4f-4513-93e4-3fa2b61f265e
  kind: Event
  lastTimestamp: "2026-09-18T19:56:26Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:55:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:26Z"
    name: message-dumper-raw-91b5b-predictor.18d68278aa595a80
    namespace: kserve-ci-e2e-test
    resourceVersion: "19172"
    uid: ebcd1c84-e31c-4448-b923-5ea191b8d976
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:56Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18684"
    uid: ff517e4b-5b4f-4513-93e4-3fa2b61f265e
  kind: Event
  lastTimestamp: "2026-09-18T19:56:26Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:55:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:26Z"
    name: message-dumper-raw-91b5b-predictor.18d68278aab91b44
    namespace: kserve-ci-e2e-test
    resourceVersion: "19173"
    uid: 089fc435-f64d-4991-b0f9-0d6be0e49dc2
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18917"
    uid: ff517e4b-5b4f-4513-93e4-3fa2b61f265e
  kind: Event
  lastTimestamp: "2026-09-18T19:56:11Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:56:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:11Z"
    name: message-dumper-raw-91b5b-predictor.18d6827c29867df4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19050"
    uid: 0e2f0b2a-94ee-4d03-8453-3535394d33c0
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:56:11Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-91b5b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18917"
    uid: ff517e4b-5b4f-4513-93e4-3fa2b61f265e
  kind: Event
  lastTimestamp: "2026-09-18T19:56:11Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:56:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:56:11Z"
    name: message-dumper-raw-91b5b-predictor.18d6827c29ed4103
    namespace: kserve-ci-e2e-test
    resourceVersion: "19052"
    uid: 2506bb6d-408e-4d99-a01c-71106cae73c4
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-91b5b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18673"
    uid: 3916628e-3b9b-46bc-b403-3bafc760308c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:51Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:51Z"
    name: message-dumper-raw-91b5b.18d682752bef53e7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18822"
    uid: 6a8ba969-3079-4642-b5b6-399790fa87ef
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-91b5b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18820"
    uid: 3916628e-3b9b-46bc-b403-3bafc760308c
  kind: Event
  lastTimestamp: "2026-09-18T19:55:51Z"
  message: InferenceService [message-dumper-raw-91b5b] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:55:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:51Z"
    name: message-dumper-raw-91b5b.18d682776ea0fe6c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18821"
    uid: 6695942e-b164-42d5-98b9-34b1b3c413df
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T20:05:01.847057Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23817"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T20:05:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T20:05:01Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7981f2f6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23824"
    uid: 672279a2-97bf-4184-8ee1-896c8ddac15f
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23829"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: Add eth0 [10.134.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7ba219213
    namespace: kserve-ci-e2e-test
    resourceVersion: "23837"
    uid: 867e3a78-1ac6-4508-877f-3792cec2e003
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: Container image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7bb529a9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23839"
    uid: 92d879e1-f4e2-44ea-88f1-4b740413cae3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7bdf309e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23841"
    uid: 48604625-8e2b-44fc-bece-5e3ac4f4d36e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7be5f1ee7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23842"
    uid: 80e95901-c799-4338-9268-c3f847466220
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7be679fb1
    namespace: kserve-ci-e2e-test
    resourceVersion: "23843"
    uid: 9c065043-27eb-4f2c-8d92-3cc5ac22c40b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7c1173bcc
    namespace: kserve-ci-e2e-test
    resourceVersion: "23844"
    uid: 4725657d-8388-4412-817a-045fbc570fcd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:05:02Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:05:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:05:02Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d682f7c183c3ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "23845"
    uid: ee1aae93-3433-4696-bb86-6c77ec929767
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:06:47Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:06:47Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d683101eba88bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "24677"
    uid: 88470954-affe-40e9-b062-a12d83f0909f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23819"
    uid: d38b9751-9f78-4e92-8f34-8592215a3d16
  kind: Event
  lastTimestamp: "2026-09-18T20:06:47Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:06:47Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4.18d683101ebb02df
    namespace: kserve-ci-e2e-test
    resourceVersion: "24678"
    uid: d24d9428-6a22-4eef-818b-0f71d26597e8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-cipn-predictor-9774bf55d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23808"
    uid: 24f82cb9-2852-4799-833c-1128a293abf7
  kind: Event
  lastTimestamp: "2026-09-18T20:05:01Z"
  message: 'Created pod: message-dumper-raw-cipn-predictor-9774bf55d-rqnx4'
  metadata:
    creationTimestamp: "2026-09-18T20:05:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:01Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d.18d682f797b5f4c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "23818"
    uid: bbd54d96-f5c6-4faa-9ed1-81341baa0675
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23806"
    uid: a7cfa1a6-5890-4d20-87c6-dc61ae3a4aee
  kind: Event
  lastTimestamp: "2026-09-18T20:05:01Z"
  message: Scaled up replica set message-dumper-raw-cipn-predictor-9774bf55d from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-18T20:05:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:01Z"
    name: message-dumper-raw-cipn-predictor.18d682f7948f6b3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23810"
    uid: c6315b47-877c-46c6-a25f-8e2fd0b8583e
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23807"
    uid: 06ff4066-16f6-41a3-aa19-ff9076e6d066
  kind: Event
  lastTimestamp: "2026-09-18T20:05:31Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:31Z"
    name: message-dumper-raw-cipn-predictor.18d682fb137da2e1
    namespace: kserve-ci-e2e-test
    resourceVersion: "24114"
    uid: 2b81b743-2425-418a-899e-625f48734433
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23807"
    uid: 06ff4066-16f6-41a3-aa19-ff9076e6d066
  kind: Event
  lastTimestamp: "2026-09-18T20:05:31Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T20:05:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:05:31Z"
    name: message-dumper-raw-cipn-predictor.18d682fb13dfe37f
    namespace: kserve-ci-e2e-test
    resourceVersion: "24115"
    uid: d89343e3-b9c8-4826-85fc-3ce505ad3a49
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23799"
    uid: 64d340a3-3e40-4a6b-9354-168986edac87
  kind: Event
  lastTimestamp: "2026-09-18T20:05:11Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T20:05:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:05:11Z"
    name: message-dumper-raw-cipn.18d682f794c15c69
    namespace: kserve-ci-e2e-test
    resourceVersion: "23915"
    uid: 59e3815e-8d88-4971-96fe-e4ec80782692
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23799"
    uid: 64d340a3-3e40-4a6b-9354-168986edac87
  kind: Event
  lastTimestamp: "2026-09-18T20:05:01Z"
  message: 'Failed to update status for InferenceService "message-dumper-raw-cipn":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cipn":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T20:05:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:05:01Z"
    name: message-dumper-raw-cipn.18d682f79537a31e
    namespace: kserve-ci-e2e-test
    resourceVersion: "23814"
    uid: a87f6cb5-5cb1-489f-8a30-4949fcd617a4
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:05:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23913"
    uid: 64d340a3-3e40-4a6b-9354-168986edac87
  kind: Event
  lastTimestamp: "2026-09-18T20:05:11Z"
  message: InferenceService [message-dumper-raw-cipn] is Ready
  metadata:
    creationTimestamp: "2026-09-18T20:05:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:05:11Z"
    name: message-dumper-raw-cipn.18d682f9db7f8d20
    namespace: kserve-ci-e2e-test
    resourceVersion: "23914"
    uid: d15bad80-c3af-4465-a609-e3aa92656c01
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:53:41.146891Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17296"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-2f453-f98d689d-r6rr4
    to ip-10-0-130-199.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:53:41Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:53:41Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d682591b3af254
    namespace: kserve-ci-e2e-test
    resourceVersion: "17301"
    uid: d30b4ae6-33c1-49da-be97-f1cebf861e85
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:41Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:41Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-2f453-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:53:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:41Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d68259274b4200
    namespace: kserve-ci-e2e-test
    resourceVersion: "17311"
    uid: 9b50a73e-07f3-4ecb-b0b6-cc8ed5a098d0
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:42Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17305"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:42Z"
  message: Add eth0 [10.132.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:53:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:53:42Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d682596192c387
    namespace: kserve-ci-e2e-test
    resourceVersion: "17323"
    uid: 2ccee20a-4611-4c2a-9c9e-cb1b0c11fcbf
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-2f453}
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:42Z"
  message: Pulling image "quay.io/opendatahub/kserve-router@sha256:341d7d999371c3925a2bf1dab18a51cbb6b284ad1c310340df58f19b3f00d99f"
  metadata:
    creationTimestamp: "2026-09-18T19:53:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:42Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d6825962d9701c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17325"
    uid: 91146c60-3a46-475c-b36a-552b66a5d0b8
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-2f453}
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:44Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:341d7d999371c3925a2bf1dab18a51cbb6b284ad1c310340df58f19b3f00d99f"
    in 1.893s (1.893s including waiting). Image size: 214866500 bytes.'
  metadata:
    creationTimestamp: "2026-09-18T19:53:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:44Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d68259d3b60415
    namespace: kserve-ci-e2e-test
    resourceVersion: "17343"
    uid: c903016d-ebba-47a4-af0f-50f46d267e7f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-2f453}
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:44Z"
  message: 'Created container: model-chainer-raw-2f453'
  metadata:
    creationTimestamp: "2026-09-18T19:53:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:44Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d68259d77c1694
    namespace: kserve-ci-e2e-test
    resourceVersion: "17344"
    uid: abac6888-6753-42f3-a8cb-e42b77e170ed
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-2f453}
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:44Z"
  message: Started container model-chainer-raw-2f453
  metadata:
    creationTimestamp: "2026-09-18T19:53:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:44Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d68259d80ac715
    namespace: kserve-ci-e2e-test
    resourceVersion: "17345"
    uid: beb6e8ab-1498-4f8b-a4a0-e5ab09406799
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-2f453}
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:53:51Z"
  message: Stopping container model-chainer-raw-2f453
  metadata:
    creationTimestamp: "2026-09-18T19:53:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:53:51Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d6825b72f1048f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17423"
    uid: ccc63a34-6b36-4d40-8961-b73cccf2f828
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-2f453}
    kind: Pod
    name: model-chainer-raw-2f453-f98d689d-r6rr4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17298"
    uid: 4e71a30e-35cd-45a0-a4ac-6caa244fd08d
  kind: Event
  lastTimestamp: "2026-09-18T19:54:20Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-18T19:53:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:54:20Z"
    name: model-chainer-raw-2f453-f98d689d-r6rr4.18d6825c83cb5900
    namespace: kserve-ci-e2e-test
    resourceVersion: "17819"
    uid: 49f7eeb5-8ff4-4c4d-ab7c-2b7ab0fd3e0e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-2f453-f98d689d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17293"
    uid: 5fc7366a-678e-4449-b58d-7def7de73f89
  kind: Event
  lastTimestamp: "2026-09-18T19:53:41Z"
  message: 'Created pod: model-chainer-raw-2f453-f98d689d-r6rr4'
  metadata:
    creationTimestamp: "2026-09-18T19:53:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:41Z"
    name: model-chainer-raw-2f453-f98d689d.18d682591ad71a34
    namespace: kserve-ci-e2e-test
    resourceVersion: "17297"
    uid: 7b1f163e-b155-4896-aae6-6519380a3508
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:41Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "17291"
    uid: f42aafd4-7780-4e8c-aded-58f16234a7f4
  kind: Event
  lastTimestamp: "2026-09-18T19:53:41Z"
  message: Scaled up replica set model-chainer-raw-2f453-f98d689d from 0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:53:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:53:41Z"
    name: model-chainer-raw-2f453.18d682591a20950d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17294"
    uid: ad2b5b63-e028-4a30-8b3d-04b249cfd986
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:53:41Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-2f453
    namespace: kserve-ci-e2e-test
    resourceVersion: "17307"
    uid: 9987b2b4-0bce-4c50-85eb-501eb3ccc8c9
  kind: Event
  lastTimestamp: "2026-09-18T19:53:41Z"
  message: InferenceGraph [model-chainer-raw-2f453] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:53:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:53:41Z"
    name: model-chainer-raw-2f453.18d682591f39a1d9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17309"
    uid: 123c78c0-f926-4f13-880d-31f8bd08c439
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:55:31.453415Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18531"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    to ip-10-0-130-199.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:55:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:55:31Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d68272ca02e57c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18536"
    uid: 6485ce04-eafa-42d4-9737-5d66cd731a3f
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18541"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: "2026-09-18T19:55:32Z"
  message: Add eth0 [10.132.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:55:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:55:32Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d68272ec667351
    namespace: kserve-ci-e2e-test
    resourceVersion: "18553"
    uid: 7f9393e3-31bf-4247-9029-9320bc0e3251
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e0026}
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18533"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: "2026-09-18T19:55:32Z"
  message: Container image "quay.io/opendatahub/kserve-router@sha256:341d7d999371c3925a2bf1dab18a51cbb6b284ad1c310340df58f19b3f00d99f"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:55:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:32Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d68272edbbba79
    namespace: kserve-ci-e2e-test
    resourceVersion: "18555"
    uid: e1b4952c-fe3f-40b4-bcf0-25291c3dab9e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e0026}
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18533"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: "2026-09-18T19:55:32Z"
  message: 'Created container: model-chainer-raw-hpa-e0026'
  metadata:
    creationTimestamp: "2026-09-18T19:55:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:32Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d68272f1445d76
    namespace: kserve-ci-e2e-test
    resourceVersion: "18556"
    uid: 13b942c2-c050-441c-a5e7-8ce75734687c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e0026}
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18533"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: "2026-09-18T19:55:32Z"
  message: Started container model-chainer-raw-hpa-e0026
  metadata:
    creationTimestamp: "2026-09-18T19:55:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:32Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d68272f1cee11e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18557"
    uid: 1451cedc-7486-4861-b805-8a2fe56c03ab
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e0026}
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18533"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: "2026-09-18T19:55:41Z"
  message: Stopping container model-chainer-raw-hpa-e0026
  metadata:
    creationTimestamp: "2026-09-18T19:55:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:55:41Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d68275222a60fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18652"
    uid: a8693c28-14bf-4145-a526-ffbe000f13d5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-e0026}
    kind: Pod
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt
    namespace: kserve-ci-e2e-test
    resourceVersion: "18533"
    uid: a05a7a38-5b47-4f33-bdbb-02b95d9ad5f5
  kind: Event
  lastTimestamp: "2026-09-18T19:56:08Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-18T19:55:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:56:08Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f-djfrt.18d682758c320ec8
    namespace: kserve-ci-e2e-test
    resourceVersion: "19028"
    uid: 70396c4c-bf55-45d8-8f49-f5c3f0c11d95
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-130-199.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-130-199.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-hpa-e0026-646bcd588f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18528"
    uid: 4a12bb21-42b0-419e-89a1-d84183b1aad9
  kind: Event
  lastTimestamp: "2026-09-18T19:55:31Z"
  message: 'Created pod: model-chainer-raw-hpa-e0026-646bcd588f-djfrt'
  metadata:
    creationTimestamp: "2026-09-18T19:55:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:31Z"
    name: model-chainer-raw-hpa-e0026-646bcd588f.18d68272c99714c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18532"
    uid: 106615a1-d68a-4b00-a220-a5f07e3308b5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-hpa-e0026
    namespace: kserve-ci-e2e-test
    resourceVersion: "18526"
    uid: d23ebc51-6e2f-4d81-821a-489897e4e6d3
  kind: Event
  lastTimestamp: "2026-09-18T19:55:31Z"
  message: Scaled up replica set model-chainer-raw-hpa-e0026-646bcd588f from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-18T19:55:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:55:31Z"
    name: model-chainer-raw-hpa-e0026.18d68272c8ea8864
    namespace: kserve-ci-e2e-test
    resourceVersion: "18529"
    uid: 5c2cee59-3ed1-471c-a1e5-90cec8f76e30
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:55:31Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-e0026
    namespace: kserve-ci-e2e-test
    resourceVersion: "18543"
    uid: accff0bf-3b28-4bad-adda-7d669446ef9b
  kind: Event
  lastTimestamp: "2026-09-18T19:55:31Z"
  message: InferenceGraph [model-chainer-raw-hpa-e0026] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:55:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:55:31Z"
    name: model-chainer-raw-hpa-e0026.18d68272ce037789
    namespace: kserve-ci-e2e-test
    resourceVersion: "18545"
    uid: 2f154be3-0026-4f69-97dd-cd069b942125
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:57:35.345461Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20004"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:57:35Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:57:35Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d6828fa28d83ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "20012"
    uid: 37533fc3-b0df-49a3-9f3a-15767e1af7b0
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:35Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:35Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-fcbfb-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-18T19:57:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:35Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d6828fb1d06628
    namespace: kserve-ci-e2e-test
    resourceVersion: "20020"
    uid: 3773c1da-8f56-4232-803e-6fc505185d94
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:36Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20016"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:36Z"
  message: Add eth0 [10.134.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:57:36Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d6828fe819ba02
    namespace: kserve-ci-e2e-test
    resourceVersion: "20030"
    uid: 42894974-3658-41e3-a106-603760bb07a6
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:36Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:36Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d6828feb17fafd
    namespace: kserve-ci-e2e-test
    resourceVersion: "20032"
    uid: 44137517-f1ef-4454-b8b0-cd3a3be8a4f7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:36Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:36Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d6828fedab21e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "20034"
    uid: 0798a22d-3527-4538-858f-c69cdef2ae8c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:36Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:36Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:57:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:36Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d6828fee193408
    namespace: kserve-ci-e2e-test
    resourceVersion: "20035"
    uid: ce5ad5d7-81ac-47aa-ba5b-972b9f692f2a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:40Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:40Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d68290e5d055aa
    namespace: kserve-ci-e2e-test
    resourceVersion: "20072"
    uid: 02e2318f-9c3b-4f2e-b037-3a47d0f40949
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:40Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:40Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d68290e8df4e67
    namespace: kserve-ci-e2e-test
    resourceVersion: "20075"
    uid: 14967157-de0e-45ac-8b98-11019002fe36
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:40Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:40Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d68290e9515332
    namespace: kserve-ci-e2e-test
    resourceVersion: "20078"
    uid: d37fa2b0-37e1-456d-b00e-abe099213a89
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:40Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:40Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d68290e95c318a
    namespace: kserve-ci-e2e-test
    resourceVersion: "20080"
    uid: 6bf363a0-237c-4c16-bbfc-666cca97d60a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:40Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:40Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d68290ec0cb9c1
    namespace: kserve-ci-e2e-test
    resourceVersion: "20082"
    uid: 9be5d65e-bf56-44ab-9377-7b0cd09211cd
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:57:40Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:57:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:57:40Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d68290ec7d67bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20083"
    uid: c45ae78d-4d3a-4368-a64c-0675e3da520d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:59:08Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:57:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:08Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d682915d9a158c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20777"
    uid: 1d321a7c-770f-4370-a080-12f027f2ed2e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:59:05Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:59:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:05Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d682a49b028067
    namespace: kserve-ci-e2e-test
    resourceVersion: "20751"
    uid: 96226891-7cab-4f0e-b473-af9c539bb32d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:59:05Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:59:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:05Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d682a49b02f9ac
    namespace: kserve-ci-e2e-test
    resourceVersion: "20752"
    uid: 70e3545e-3ed4-4850-9872-912f4593c6aa
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20008"
    uid: 425bd21e-f06d-402f-8096-7af036a48170
  kind: Event
  lastTimestamp: "2026-09-18T19:59:08Z"
  message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp
    10.134.0.29:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T19:59:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:08Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk.18d682a563c7cc64
    namespace: kserve-ci-e2e-test
    resourceVersion: "20776"
    uid: 32122798-b53f-485e-af58-e79063ee7d3e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:35Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55
    namespace: kserve-ci-e2e-test
    resourceVersion: "19996"
    uid: 2d198425-921a-46be-8fc3-58faa2c9d10a
  kind: Event
  lastTimestamp: "2026-09-18T19:57:35Z"
  message: 'Created pod: raw-sklearn-fcbfb-predictor-58ccc8ff55-jlbtk'
  metadata:
    creationTimestamp: "2026-09-18T19:57:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:35Z"
    name: raw-sklearn-fcbfb-predictor-58ccc8ff55.18d6828fa175287e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20005"
    uid: 64e7413c-66cc-4d0f-b67c-adcaef859abd
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:35Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-fcbfb-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19993"
    uid: 9ee3e4ff-4e44-4d8a-b3b9-652e229425ef
  kind: Event
  lastTimestamp: "2026-09-18T19:57:35Z"
  message: Scaled up replica set raw-sklearn-fcbfb-predictor-58ccc8ff55 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-18T19:57:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:57:35Z"
    name: raw-sklearn-fcbfb-predictor.18d6828f9e3d4042
    namespace: kserve-ci-e2e-test
    resourceVersion: "20000"
    uid: f858b6b2-f658-4925-96d7-72c48156030f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:50Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-fcbfb-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19994"
    uid: dd6b0ef7-39c9-42b6-89e9-4404115b68d3
  kind: Event
  lastTimestamp: "2026-09-18T19:58:05Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:57:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:58:05Z"
    name: raw-sklearn-fcbfb-predictor.18d682931c72a8d9
    namespace: kserve-ci-e2e-test
    resourceVersion: "20277"
    uid: 903b1505-9bf8-4810-95bc-bd52597c45d4
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:50Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-fcbfb-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19994"
    uid: dd6b0ef7-39c9-42b6-89e9-4404115b68d3
  kind: Event
  lastTimestamp: "2026-09-18T19:58:05Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:57:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:58:05Z"
    name: raw-sklearn-fcbfb-predictor.18d682931cd2e806
    namespace: kserve-ci-e2e-test
    resourceVersion: "20278"
    uid: 4956775a-c3fc-4eab-8287-56bf4821905c
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:58:20Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-fcbfb-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20160"
    uid: dd6b0ef7-39c9-42b6-89e9-4404115b68d3
  kind: Event
  lastTimestamp: "2026-09-18T19:59:05Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:58:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:05Z"
    name: raw-sklearn-fcbfb-predictor.18d6829a1a1d113b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20734"
    uid: b9aba15d-17d5-4d63-8411-995f4d3cfc8d
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-18T19:58:20Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-fcbfb-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20160"
    uid: dd6b0ef7-39c9-42b6-89e9-4404115b68d3
  kind: Event
  lastTimestamp: "2026-09-18T19:59:05Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:58:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:05Z"
    name: raw-sklearn-fcbfb-predictor.18d6829a1a7a0038
    namespace: kserve-ci-e2e-test
    resourceVersion: "20735"
    uid: 4488da04-5074-4783-8726-979cb39f3904
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:35Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-fcbfb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19986"
    uid: 4673522e-6dd0-49b4-88ef-ef8c1f9632c6
  kind: Event
  lastTimestamp: "2026-09-18T19:58:58Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:57:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:58:58Z"
    name: raw-sklearn-fcbfb.18d6828f9e40f847
    namespace: kserve-ci-e2e-test
    resourceVersion: "20684"
    uid: 1d8ce821-9e9c-4488-8620-943ca0bba613
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:57:35Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-fcbfb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19986"
    uid: 4673522e-6dd0-49b4-88ef-ef8c1f9632c6
  kind: Event
  lastTimestamp: "2026-09-18T19:57:35Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-fcbfb": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-fcbfb":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:57:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:57:35Z"
    name: raw-sklearn-fcbfb.18d6828f9edc2d8e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20001"
    uid: 020f61f3-fd9d-4528-bf95-87fd41880bd1
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:58:58Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-fcbfb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20681"
    uid: 4673522e-6dd0-49b4-88ef-ef8c1f9632c6
  kind: Event
  lastTimestamp: "2026-09-18T19:58:58Z"
  message: InferenceService [raw-sklearn-fcbfb] is Ready
  metadata:
    creationTimestamp: "2026-09-18T19:58:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:58:58Z"
    name: raw-sklearn-fcbfb.18d682a3150d7cbd
    namespace: kserve-ci-e2e-test
    resourceVersion: "20682"
    uid: c5bf3aa7-90f2-4dd0-9d07-5db378256c79
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-18T19:59:10.582609Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20816"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    to ip-10-0-141-190.ec2.internal
  metadata:
    creationTimestamp: "2026-09-18T19:59:10Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-18T19:59:10Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a5cf211016
    namespace: kserve-ci-e2e-test
    resourceVersion: "20820"
    uid: 86361b4e-9a96-4f34-a478-651a6e8b7bf8
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-65774549c6-lqwf5
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20824"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:11Z"
  message: Add eth0 [10.134.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-18T19:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-18T19:59:11Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a5f0eaf3e2
    namespace: kserve-ci-e2e-test
    resourceVersion: "20835"
    uid: 5d734fdf-b693-41f2-bfcc-0bd27a1b6cb8
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:11Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:1fa7c8d29012cdb239c749146593a8aa2685db50366be2fc7d147daf97857e63"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:11Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a5f22dfab1
    namespace: kserve-ci-e2e-test
    resourceVersion: "20837"
    uid: 95299efc-6160-4981-ae6d-4cc2096389bb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:11Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-18T19:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:11Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a5f5108134
    namespace: kserve-ci-e2e-test
    resourceVersion: "20838"
    uid: ce7e0cb2-1abe-4ab2-837e-7f88dc07d780
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:11Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-18T19:59:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:11Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a5f59274d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "20839"
    uid: 28b085b9-5dff-407c-8dcd-9ca9c39733bd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:15Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:59:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:15Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a6ddea2428
    namespace: kserve-ci-e2e-test
    resourceVersion: "20870"
    uid: bf6c20b5-0d23-4d85-a967-eacfc2c6b6ed
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:15Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-18T19:59:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:15Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a6e121da85
    namespace: kserve-ci-e2e-test
    resourceVersion: "20873"
    uid: 7533a6bf-b19a-4484-8493-31a99aae3bf4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:15Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T19:59:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:15Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a6e18fea3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20874"
    uid: b3d2f3d5-b101-429a-8886-ad0ec4490b85
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:15Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-18T19:59:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:15Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a6e199aa62
    namespace: kserve-ci-e2e-test
    resourceVersion: "20876"
    uid: ebb079a7-2097-4924-aa44-dea4060aa443
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:15Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-18T19:59:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:15Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a6e40f0346
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: a087a8fd-038c-46dd-8df1-d5ce38377640
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T19:59:15Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T19:59:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T19:59:15Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a6e47a5d2d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20879"
    uid: f4a43efe-d4cc-4d2e-80e6-a9b97bb2713a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T20:00:43Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-18T19:59:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:43Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682a755b9f3cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "21697"
    uid: 81b81048-abb4-4d61-ac7f-9e35579e4a73
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T20:00:40Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-18T20:00:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:40Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682bac5941e36
    namespace: kserve-ci-e2e-test
    resourceVersion: "21676"
    uid: ae30b3de-8abc-4651-8cd6-bd810b85d845
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T20:00:40Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-18T20:00:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:40Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682bac594a535
    namespace: kserve-ci-e2e-test
    resourceVersion: "21677"
    uid: 92c40134-eef4-4ba1-b40a-f7a25b1825cc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m
    namespace: kserve-ci-e2e-test
    resourceVersion: "20818"
    uid: 19c05583-2a24-492b-8d80-f9022c3804ff
  kind: Event
  lastTimestamp: "2026-09-18T20:00:43Z"
  message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp
    10.134.0.30:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-18T20:00:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-18T20:00:43Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m.18d682bb5be3956d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21696"
    uid: e5bf4e64-fbd7-4492-8570-0db25bea9b99
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-190.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-190.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:10Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7
    namespace: kserve-ci-e2e-test
    resourceVersion: "20805"
    uid: 08b12036-dcb9-48de-89d4-0772c81ad003
  kind: Event
  lastTimestamp: "2026-09-18T19:59:10Z"
  message: 'Created pod: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7-6ls7m'
  metadata:
    creationTimestamp: "2026-09-18T19:59:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:10Z"
    name: raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7.18d682a5cebd5901
    namespace: kserve-ci-e2e-test
    resourceVersion: "20817"
    uid: e29598f8-0393-4318-87a0-66d6c11aaed4
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:10Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-runtime-aa327-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20802"
    uid: d04de549-2aeb-4dd0-b0a0-35ca717020f4
  kind: Event
  lastTimestamp: "2026-09-18T19:59:10Z"
  message: Scaled up replica set raw-sklearn-runtime-aa327-predictor-7dfdcb6cb7 from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-18T19:59:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:10Z"
    name: raw-sklearn-runtime-aa327-predictor.18d682a5cac2580b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20807"
    uid: 94a0f932-2944-49df-8384-460dde05da35
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:25Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-aa327-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20804"
    uid: 8cf049cb-fe54-4411-a063-8087ba6267a4
  kind: Event
  lastTimestamp: "2026-09-18T19:59:55Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:59:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:55Z"
    name: raw-sklearn-runtime-aa327-predictor.18d682a949d61cc1
    namespace: kserve-ci-e2e-test
    resourceVersion: "21183"
    uid: 6eb29258-6a44-44f9-b079-efaf60da5134
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:25Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-aa327-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20804"
    uid: 8cf049cb-fe54-4411-a063-8087ba6267a4
  kind: Event
  lastTimestamp: "2026-09-18T19:59:55Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-18T19:59:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T19:59:55Z"
    name: raw-sklearn-runtime-aa327-predictor.18d682a94a40af9e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21184"
    uid: 6b60c4c6-e909-4f14-93aa-f2145f01198c
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-aa327-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20960"
    uid: 8cf049cb-fe54-4411-a063-8087ba6267a4
  kind: Event
  lastTimestamp: "2026-09-18T20:00:25Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:59:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:25Z"
    name: raw-sklearn-runtime-aa327-predictor.18d682acc8e3fb9c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21552"
    uid: 3802028b-85d4-4724-a75e-d3c8259723d6
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-aa327-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20960"
    uid: 8cf049cb-fe54-4411-a063-8087ba6267a4
  kind: Event
  lastTimestamp: "2026-09-18T20:00:25Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-18T19:59:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-18T20:00:25Z"
    name: raw-sklearn-runtime-aa327-predictor.18d682acc94256fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21553"
    uid: b03cb097-c8eb-4d33-be7b-d46c8cc527e1
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:10Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-aa327
    namespace: kserve-ci-e2e-test
    resourceVersion: "20795"
    uid: b0ad4344-41f8-4a8e-a4b4-31fe28019bbd
  kind: Event
  lastTimestamp: "2026-09-18T20:00:33Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-18T19:59:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:00:33Z"
    name: raw-sklearn-runtime-aa327.18d682a5cac6988a
    namespace: kserve-ci-e2e-test
    resourceVersion: "21609"
    uid: c3183c7e-0c40-4252-93d9-8bb1057b5418
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T19:59:10Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-aa327
    namespace: kserve-ci-e2e-test
    resourceVersion: "20795"
    uid: b0ad4344-41f8-4a8e-a4b4-31fe28019bbd
  kind: Event
  lastTimestamp: "2026-09-18T19:59:10Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-runtime-aa327":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-aa327":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-18T19:59:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T19:59:10Z"
    name: raw-sklearn-runtime-aa327.18d682a5cb357364
    namespace: kserve-ci-e2e-test
    resourceVersion: "20810"
    uid: de4f76e5-f342-417b-bead-eb1d08512563
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-18T20:00:33Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-aa327
    namespace: kserve-ci-e2e-test
    resourceVersion: "21607"
    uid: b0ad4344-41f8-4a8e-a4b4-31fe28019bbd
  kind: Event
  lastTimestamp: "2026-09-18T20:00:33Z"
  message: InferenceService [raw-sklearn-runtime-aa327] is Ready
  metadata:
    creationTimestamp: "2026-09-18T20:00:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-18T20:00:33Z"
    name: raw-sklearn-runtime-aa327.18d682b90cead260
    namespace: kserve-ci-e2e-test
    resourceVersion: "21608"
    uid: c822f874-b1b1-47c0-a206-552a96899ad1
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
kind: EventList
metadata:
  resourceVersion: "24927"