---
apiVersion: v1
items:
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:17:01.894300Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20893"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-5956b77f85-kjzjk
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7e4b87f3fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20897"
    uid: b13c57d7-5b6d-4e74-b478-18addfe47462
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20902"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Add eth0 [10.134.0.37/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7e6d7068bd
    namespace: kserve-ci-e2e-test
    resourceVersion: "20923"
    uid: fb5a663f-9d2a-4a07-b794-61ab93669a8c
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7e6eef8f9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20925"
    uid: ccfbeb26-6b07-44d6-8b6c-045623a77b82
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7e72f47744
    namespace: kserve-ci-e2e-test
    resourceVersion: "20928"
    uid: e977d288-ed4e-4f21-b666-41e1a8c3e4d8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7e73abdfda
    namespace: kserve-ci-e2e-test
    resourceVersion: "20929"
    uid: 7aea2192-4d34-43bb-b0db-814cad386907
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7f89c46d83
    namespace: kserve-ci-e2e-test
    resourceVersion: "21013"
    uid: 0893fcb3-41c6-4472-bb49-09b3c3dc1f96
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7f8dc8ff19
    namespace: kserve-ci-e2e-test
    resourceVersion: "21020"
    uid: 11f5dae9-d75f-4da8-88b8-4b69f321a267
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7f8e64a3f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21021"
    uid: b4aace0f-4fe5-4511-a371-b23d030f7451
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7f8e6e92fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21023"
    uid: d53b6d96-860e-4d42-b6ab-17962cdd9cdd
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7f91d95fd5
    namespace: kserve-ci-e2e-test
    resourceVersion: "21028"
    uid: 84cfa530-740f-4b07-876b-2d709d192da2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7f92502990
    namespace: kserve-ci-e2e-test
    resourceVersion: "21030"
    uid: 0ff68e68-124f-4b65-b5f6-ba8c7360b90f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:18:34Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:17:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:34Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e7fc5e480c7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21734"
    uid: 8f74f97d-fe4b-4434-805f-c2d849f960b5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:18:31Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:18:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:31Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e933fdb06f1
    namespace: kserve-ci-e2e-test
    resourceVersion: "21713"
    uid: cd61a1ae-ea0c-4623-8b4f-c92ea0069348
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:18:31Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:18:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:31Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e933fdc3d68
    namespace: kserve-ci-e2e-test
    resourceVersion: "21714"
    uid: 723d9770-099a-4ecf-a752-15770089b5ee
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-kjzjk
    namespace: kserve-ci-e2e-test
    resourceVersion: "20895"
    uid: 55e57a98-c38f-4ea2-b8e8-25d876d21775
  kind: Event
  lastTimestamp: "2026-09-17T18:18:34Z"
  message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp
    10.134.0.37:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:18:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:34Z"
    name: isvc-canary-create-predictor-5956b77f85-kjzjk.18d62e93cc3b5168
    namespace: kserve-ci-e2e-test
    resourceVersion: "21733"
    uid: 3086aedf-e61b-430b-9ded-0020cf7ed785
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:17:01.852590Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20885"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-predictor-5956b77f85-mvn7z
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7e490b8677
    namespace: kserve-ci-e2e-test
    resourceVersion: "20890"
    uid: 9a0878cd-c677-40b2-84d2-3e60c0514e83
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20892"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Add eth0 [10.133.0.41/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7e6afc2ac7
    namespace: kserve-ci-e2e-test
    resourceVersion: "20920"
    uid: f3a23fb3-9874-45d3-baec-666d2b95a06d
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7e6c8f6d19
    namespace: kserve-ci-e2e-test
    resourceVersion: "20922"
    uid: 71d6bf5d-6192-4556-80f9-d33f1c766957
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7e6fa68a0e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20926"
    uid: d68fd9e2-ef0e-4760-8c0e-1dc493c32d94
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7e701f852e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20927"
    uid: 9352577d-df85-4638-8fdf-ea29aada32fa
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7f9a9b94ed
    namespace: kserve-ci-e2e-test
    resourceVersion: "21032"
    uid: a98dfe98-0b1a-4ea1-8560-b8f7dcdde044
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7f9e12c6ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "21035"
    uid: 8a22049c-5432-498d-81fd-b9840ce62991
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7f9e8d7ba2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21037"
    uid: 3660a504-28fd-4ed1-9593-b3211593ca3b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7f9e996650
    namespace: kserve-ci-e2e-test
    resourceVersion: "21038"
    uid: e5b04bb1-594e-4fca-9a90-0b5e9512f468
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7fa27cee38
    namespace: kserve-ci-e2e-test
    resourceVersion: "21039"
    uid: 02ba218d-366b-48a3-b851-42f564c3fd17
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e7fa2f9bcc4
    namespace: kserve-ci-e2e-test
    resourceVersion: "21040"
    uid: 33c8bfb3-f3d4-4950-8ab6-d0cf5f75c01f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:18:15Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:17:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:15Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e8012860c39
    namespace: kserve-ci-e2e-test
    resourceVersion: "21528"
    uid: e12cd553-c250-49c8-b32b-8e2667f75bc9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:18:28Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:18:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:28Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e926250744b
    namespace: kserve-ci-e2e-test
    resourceVersion: "21652"
    uid: a4a2bca4-bfea-45cc-9963-f75f89ed5c24
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:28Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:18:28Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:18:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:28Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e926250f99d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21654"
    uid: 612f7aca-86a7-4821-a6fb-8035b038148c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-predictor-5956b77f85-mvn7z
    namespace: kserve-ci-e2e-test
    resourceVersion: "20887"
    uid: 29f5f9f4-7677-45ea-8b56-edc12b09a717
  kind: Event
  lastTimestamp: "2026-09-17T18:18:30Z"
  message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp
    10.133.0.41:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:18:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:30Z"
    name: isvc-canary-create-predictor-5956b77f85-mvn7z.18d62e92eede585f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21681"
    uid: 2e48d1f7-3b33-4192-8e58-9972e3c9ef20
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-5956b77f85
    namespace: kserve-ci-e2e-test
    resourceVersion: "20869"
    uid: fe57c5e6-00cb-418e-bb2e-cc7665410e44
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: 'Created pod: isvc-canary-create-predictor-5956b77f85-mvn7z'
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-predictor-5956b77f85.18d62e7e48a0358b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20886"
    uid: 9e1809e0-89fc-4517-a535-d8c569d97ad1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-5956b77f85
    namespace: kserve-ci-e2e-test
    resourceVersion: "20869"
    uid: fe57c5e6-00cb-418e-bb2e-cc7665410e44
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: 'Created pod: isvc-canary-create-predictor-5956b77f85-kjzjk'
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-predictor-5956b77f85.18d62e7e4b1ae152
    namespace: kserve-ci-e2e-test
    resourceVersion: "20894"
    uid: a05cd9eb-06a3-4181-86bd-f7690ef60f6a
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:28Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-predictor-5956b77f85
    namespace: kserve-ci-e2e-test
    resourceVersion: "21644"
    uid: fe57c5e6-00cb-418e-bb2e-cc7665410e44
  kind: Event
  lastTimestamp: "2026-09-17T18:18:28Z"
  message: 'Deleted pod: isvc-canary-create-predictor-5956b77f85-mvn7z'
  metadata:
    creationTimestamp: "2026-09-17T18:18:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:28Z"
    name: isvc-canary-create-predictor-5956b77f85.18d62e9262282ec9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21647"
    uid: 0aecba1e-5a6d-4a07-b56a-8180759dfcae
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20868"
    uid: 6b22307b-4bb7-40d8-a627-76d47ee8cfc1
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: Scaled up replica set isvc-canary-create-predictor-5956b77f85 from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-predictor.18d62e7e4582c10e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20871"
    uid: f0fb7850-ff70-46de-a68f-5f6cff1995d7
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:28Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21640"
    uid: 6b22307b-4bb7-40d8-a627-76d47ee8cfc1
  kind: Event
  lastTimestamp: "2026-09-17T18:18:28Z"
  message: Scaled down replica set isvc-canary-create-predictor-5956b77f85 from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-17T18:18:28Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:28Z"
    name: isvc-canary-create-predictor.18d62e9261b9fe9f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21645"
    uid: 48e28812-308b-401f-9c70-1580dd02dace
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:17:01.818032Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20875"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7e46fc2e93
    namespace: kserve-ci-e2e-test
    resourceVersion: "20880"
    uid: a37ed442-fbb8-4097-82ac-02a5bbc8bcf9
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-create-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7e54ad7b1d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20909"
    uid: da5e33de-d3bd-4012-9497-6e00d89efa00
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20884"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Add eth0 [10.134.0.36/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7e8cf80c3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "20941"
    uid: 118a7fe1-e146-49bb-bb19-c43ff083f4b1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:03Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:03Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7e8e9e8548
    namespace: kserve-ci-e2e-test
    resourceVersion: "20943"
    uid: 89cd83e1-5a19-457e-a762-376b45277ce0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:03Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:17:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:03Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7e92958edc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20945"
    uid: f6d7df03-7bb7-4429-a74b-89088e306389
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:03Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:17:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:03Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7e930f69ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "20946"
    uid: a63c1770-3d9d-4a61-a532-1def5fe7e6ff
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7f89e4d28f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21014"
    uid: 099e8cbe-4d4c-4c8f-a58f-f407c377f377
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7f8dbba211
    namespace: kserve-ci-e2e-test
    resourceVersion: "21019"
    uid: 73e402b4-4440-4816-bbcd-d48e5e6b659d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7f8e64bdc2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21022"
    uid: 21fa984b-c9d4-458b-8a53-7fe7a18b50d2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7f8e6fe921
    namespace: kserve-ci-e2e-test
    resourceVersion: "21024"
    uid: 804c05b0-a58f-4645-af05-e5ce33837440
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7f91cacc6e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21027"
    uid: 16ead02b-eb6b-4614-96c4-19a61e54e9f6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:17:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:17:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:07Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e7f92410d24
    namespace: kserve-ci-e2e-test
    resourceVersion: "21029"
    uid: 5a541def-91fa-4157-89e4-d12791ab57a6
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:18:18Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:17:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:18Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e8001d91225
    namespace: kserve-ci-e2e-test
    resourceVersion: "21546"
    uid: 6f8e60ad-b4f4-4d16-8230-4f4f1dd868e4
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:18:31Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:18:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:31Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e933f93d4f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21710"
    uid: bfca528c-4223-4bb5-9bd0-9682514f828b
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:18:31Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:18:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:31Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e933f955402
    namespace: kserve-ci-e2e-test
    resourceVersion: "21711"
    uid: 17fa1e64-c4be-4757-a432-1629bb45ecd0
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf
    namespace: kserve-ci-e2e-test
    resourceVersion: "20878"
    uid: 5ea4957c-61b3-4888-a197-db7ddefbd6ea
  kind: Event
  lastTimestamp: "2026-09-17T18:18:35Z"
  message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp
    10.134.0.36:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:18:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:35Z"
    name: isvc-canary-create-v2-predictor-5bb585469b-kqcgf.18d62e9408398b53
    namespace: kserve-ci-e2e-test
    resourceVersion: "21743"
    uid: 2e7a329e-b0d2-4411-97b5-77a5046e5c8f
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-create-v2-predictor-5bb585469b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20860"
    uid: 9ab514bd-055f-4beb-bbec-a4804eab5f46
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: 'Created pod: isvc-canary-create-v2-predictor-5bb585469b-kqcgf'
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-v2-predictor-5bb585469b.18d62e7e468943ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "20877"
    uid: dd361071-7549-4816-b66b-f27006490596
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-create-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20859"
    uid: e50a0f1b-85b8-48ce-833f-f5e87f06159b
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: Scaled up replica set isvc-canary-create-v2-predictor-5bb585469b from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create-v2-predictor.18d62e7e43987040
    namespace: kserve-ci-e2e-test
    resourceVersion: "20861"
    uid: 520ceecb-2dbf-4636-a800-0f79179d27b2
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 23
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20852"
    uid: ee00c905-a892-436b-84d5-94e4bb81e8b1
  kind: Event
  lastTimestamp: "2026-09-17T18:18:28Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:18:28Z"
    name: isvc-canary-create.18d62e7e452ea849
    namespace: kserve-ci-e2e-test
    resourceVersion: "21662"
    uid: 2590c8a6-33f7-4146-969d-e719091f5de2
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20852"
    uid: ee00c905-a892-436b-84d5-94e4bb81e8b1
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: 'Failed to update status for InferenceService "isvc-canary-create": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-create":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create.18d62e7e4595bb2c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20873"
    uid: e7ecc18c-7400-4fc8-aa06-99ea1c44e2d4
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "20852"
    uid: ee00c905-a892-436b-84d5-94e4bb81e8b1
  kind: Event
  lastTimestamp: "2026-09-17T18:17:01Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-create": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:17:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:17:01Z"
    name: isvc-canary-create.18d62e7e4596127f
    namespace: kserve-ci-e2e-test
    resourceVersion: "20874"
    uid: f86f9414-b98c-4219-b35d-019bddf9c03d
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-create
    namespace: kserve-ci-e2e-test
    resourceVersion: "21616"
    uid: ee00c905-a892-436b-84d5-94e4bb81e8b1
  kind: Event
  lastTimestamp: "2026-09-17T18:18:25Z"
  message: InferenceService [isvc-canary-create] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:18:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:18:25Z"
    name: isvc-canary-create.18d62e91ca9956f8
    namespace: kserve-ci-e2e-test
    resourceVersion: "21617"
    uid: e1b8a632-4c90-4785-9eaf-55db7cae4944
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:18:37.086595Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21784"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-7877c757df-4mrdt
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e94756f0b34
    namespace: kserve-ci-e2e-test
    resourceVersion: "21794"
    uid: 9d88120d-dd2a-4510-8b31-072ac7a33408
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e9483c4565e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21819"
    uid: 0f4daa86-ee5a-4fe0-b66a-8622a99d8f0b
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21800"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: Add eth0 [10.133.0.43/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e94bc138983
    namespace: kserve-ci-e2e-test
    resourceVersion: "21846"
    uid: fe92212a-7a34-4d03-9aba-89a83e329926
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e94bd8b80e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "21848"
    uid: dfcaa167-990d-4d46-a573-a7e14e406fc8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e94c23e620f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21852"
    uid: 2a638694-a287-46a5-aaed-fa7be6412f76
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e94c2b6e64f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21853"
    uid: 76333e45-6e90-42b8-a665-935799880d05
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e95d4215b04
    namespace: kserve-ci-e2e-test
    resourceVersion: "21914"
    uid: a776d9e6-1bf5-4df2-a9f2-012c48a4a57c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:43Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:18:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:43Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e95d8af84d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21920"
    uid: 32a53af2-108e-4594-88a0-e4743a458678
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:43Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:18:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:43Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e95d91f001f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21921"
    uid: 3ccee902-8f1e-4aed-8bc1-92d1a888e2ac
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:43Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:43Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e95d92ba159
    namespace: kserve-ci-e2e-test
    resourceVersion: "21922"
    uid: 8e79012c-2fc5-4cbd-a43f-0e6e02669e58
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:43Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:18:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:43Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e95dbee32b7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21924"
    uid: c0ed6305-c644-4595-9f7b-48ba0d25e7b8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:18:43Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:18:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:43Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e95dc586e76
    namespace: kserve-ci-e2e-test
    resourceVersion: "21925"
    uid: b2ab2da4-2f20-4e1c-b7ad-65c90747b0f9
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:43Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:19:49Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:18:43Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:19:49Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62e96105a3bf7
    namespace: kserve-ci-e2e-test
    resourceVersion: "22404"
    uid: eb714b0e-26c3-4856-921d-c6a5d0da86ef
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:19:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:19:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62ea78e03e135
    namespace: kserve-ci-e2e-test
    resourceVersion: "22482"
    uid: ea48b59b-c103-4817-ac31-98365501a2be
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:19:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:19:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62ea78e04bc34
    namespace: kserve-ci-e2e-test
    resourceVersion: "22483"
    uid: 4dbce45c-df19-4c4d-8c75-15ee1ac3230f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:19:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-4mrdt
    namespace: kserve-ci-e2e-test
    resourceVersion: "21788"
    uid: b2df4540-d09d-45b4-aa30-a303757392d8
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp
    10.133.0.43:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:19:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote-predictor-7877c757df-4mrdt.18d62ea7c287f4d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "22503"
    uid: d6e6242b-82ee-4891-89b8-3567acac9c3b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:18:37.137533Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21799"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-predictor-7877c757df-czzhv
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e94787855df
    namespace: kserve-ci-e2e-test
    resourceVersion: "21804"
    uid: 87550498-0620-400c-953d-4ce537697fc5
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-promote-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e947fc7a661
    namespace: kserve-ci-e2e-test
    resourceVersion: "21815"
    uid: 36b4bbcb-4d35-4503-a450-f8a54c3876a1
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21809"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: Add eth0 [10.134.0.38/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e94af877b31
    namespace: kserve-ci-e2e-test
    resourceVersion: "21841"
    uid: 09c723d7-c3a7-471e-b756-0a3ba0fc7f7f
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e94b0f5d125
    namespace: kserve-ci-e2e-test
    resourceVersion: "21843"
    uid: 036c2efa-368a-4c3b-88b8-0fba6db28a3a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e94b3e6ad22
    namespace: kserve-ci-e2e-test
    resourceVersion: "21844"
    uid: 48c480de-5837-478b-80c8-6658c7cb9d87
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:38Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:18:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:38Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e94b4737c28
    namespace: kserve-ci-e2e-test
    resourceVersion: "21845"
    uid: 8b373174-0d70-4be0-991c-3e01915fdbb0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e95c10a6336
    namespace: kserve-ci-e2e-test
    resourceVersion: "21905"
    uid: 3a6001f3-c702-43ca-8723-adbb3c24fd86
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e95c4ea2ce9
    namespace: kserve-ci-e2e-test
    resourceVersion: "21908"
    uid: f8d01a3f-4a36-4237-b356-54698a710334
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e95c5731435
    namespace: kserve-ci-e2e-test
    resourceVersion: "21909"
    uid: f145ebc7-2a99-408e-9a70-6c0d291e28d6
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e95c57f5b9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21910"
    uid: 448bcdc3-a58d-450a-a5ee-12cd6d2da8b5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e95c863c077
    namespace: kserve-ci-e2e-test
    resourceVersion: "21912"
    uid: c3af1002-388f-4f13-858c-8abfe5dfe0f4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e95c8e7e3dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "21913"
    uid: e5638bee-a889-4411-af39-5a803d08a9d2
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:20:10Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:18:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:10Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62e9638fe7167
    namespace: kserve-ci-e2e-test
    resourceVersion: "22629"
    uid: b2881993-3afb-4574-833d-b0e7df231b26
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62ea96f63b5ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "22585"
    uid: 951d0bfb-e211-4e5a-9412-dea4e11b8078
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62ea96f6449ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "22586"
    uid: a5b4125b-be6f-401b-b140-bf90d073495d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-predictor-7877c757df-czzhv
    namespace: kserve-ci-e2e-test
    resourceVersion: "21802"
    uid: 5e6a2de7-9e3c-4387-9ec4-127ed99cb4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:20:10Z"
  message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp
    10.134.0.38:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:20:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:10Z"
    name: isvc-canary-promote-predictor-7877c757df-czzhv.18d62eaa3f3410b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "22628"
    uid: 1d674fef-15d6-4c91-a3d1-bbb7f27a4160
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-7877c757df
    namespace: kserve-ci-e2e-test
    resourceVersion: "21776"
    uid: 2786fdc5-b159-4080-9aaa-2adfc0651b23
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: 'Created pod: isvc-canary-promote-predictor-7877c757df-4mrdt'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor-7877c757df.18d62e9475152d6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21791"
    uid: a1c21773-8c02-43ab-89c2-d10c34308d86
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-7877c757df
    namespace: kserve-ci-e2e-test
    resourceVersion: "21776"
    uid: 2786fdc5-b159-4080-9aaa-2adfc0651b23
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: 'Created pod: isvc-canary-promote-predictor-7877c757df-czzhv'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor-7877c757df.18d62e94780f9c4c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21801"
    uid: fddd6c65-585d-4c28-8344-01add6aa79b6
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:19:59Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-predictor-7877c757df
    namespace: kserve-ci-e2e-test
    resourceVersion: "22474"
    uid: 2786fdc5-b159-4080-9aaa-2adfc0651b23
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: 'Deleted pod: isvc-canary-promote-predictor-7877c757df-4mrdt'
  metadata:
    creationTimestamp: "2026-09-17T18:19:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote-predictor-7877c757df.18d62ea78dcff7e9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22477"
    uid: 969cf963-20b1-46da-9f79-c5840ca7b48a
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21775"
    uid: fa95f013-b0e2-4230-9469-46a3a4461df4
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: Scaled up replica set isvc-canary-promote-predictor-7877c757df from 0 to
    2
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-predictor.18d62e9472155982
    namespace: kserve-ci-e2e-test
    resourceVersion: "21778"
    uid: fb3bd43b-d46a-47cb-a0c1-6572c60e6d4b
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:19:59Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22470"
    uid: fa95f013-b0e2-4230-9469-46a3a4461df4
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: Scaled down replica set isvc-canary-promote-predictor-7877c757df from 2
    to 1
  metadata:
    creationTimestamp: "2026-09-17T18:19:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote-predictor.18d62ea78d61fd85
    namespace: kserve-ci-e2e-test
    resourceVersion: "22475"
    uid: a78f8b45-85d9-40bd-8b9d-572213964a87
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:20:07.249449Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22589"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ea9738f0fe4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22593"
    uid: 120d2005-8427-4441-808b-82ae5e36408c
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22597"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: Add eth0 [10.134.0.39/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ea9959b38a6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22599"
    uid: 7197ba6a-b863-4e81-a60b-44e260338977
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ea9971717e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "22601"
    uid: dbeee09a-369e-40d6-8d78-861e1f209feb
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ea999d50189
    namespace: kserve-ci-e2e-test
    resourceVersion: "22602"
    uid: 96534f61-9b04-4520-b5a5-3bc1594bfcc7
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ea99a474e9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "22603"
    uid: d078e9a5-1ee4-407f-9690-41a9298d02b7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:12Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:20:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:12Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaa8f72e6c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "22638"
    uid: 809d0f2f-6067-487f-9165-20bbbefe27fc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:12Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:20:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:12Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaa92a5a187
    namespace: kserve-ci-e2e-test
    resourceVersion: "22643"
    uid: 61332897-03f5-46ac-88a3-d473e84f3b0a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:12Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:20:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:12Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaa9316c14c
    namespace: kserve-ci-e2e-test
    resourceVersion: "22644"
    uid: 30f76bf0-0af0-4b50-87de-7a0be1c8fce4
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:12Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:20:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:12Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaa93204d64
    namespace: kserve-ci-e2e-test
    resourceVersion: "22645"
    uid: f54a2516-b7eb-4da3-ab84-ef0613f3eed4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:12Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:20:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:12Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaa9547b531
    namespace: kserve-ci-e2e-test
    resourceVersion: "22646"
    uid: 06c352d3-6353-4a09-8fdf-f662fc87e410
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:20:12Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:20:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:20:12Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaa95fd4573
    namespace: kserve-ci-e2e-test
    resourceVersion: "22647"
    uid: 6d1bf72f-9cd8-40e3-ae63-b8740260a513
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:21:39Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:20:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:39Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62eaacb9b5418
    namespace: kserve-ci-e2e-test
    resourceVersion: "23270"
    uid: 05d88a6a-92e7-4a93-aab5-b65f9325627b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:21:37Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:21:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ebe72d407f9
    namespace: kserve-ci-e2e-test
    resourceVersion: "23253"
    uid: 770a5641-e310-4df9-b9aa-1799654f4fed
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:21:37Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:21:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ebe72d4a105
    namespace: kserve-ci-e2e-test
    resourceVersion: "23255"
    uid: 5661ac2d-1c5c-4f58-8f22-77b599843eb0
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:39Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg
    namespace: kserve-ci-e2e-test
    resourceVersion: "22591"
    uid: 5191a2c0-f6c5-4f46-9264-59f528fc57e6
  kind: Event
  lastTimestamp: "2026-09-17T18:21:39Z"
  message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp
    10.134.0.39:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:21:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:39Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg.18d62ebed1dee1cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "23269"
    uid: 702b4bfa-2d46-4ab5-91fd-f1a301902909
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:18:37.085321Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21783"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e94755ba2f6
    namespace: kserve-ci-e2e-test
    resourceVersion: "21793"
    uid: 26bbd7e5-5c50-4bba-98a8-45379ec66257
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21798"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: Add eth0 [10.133.0.42/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e9497ade55c
    namespace: kserve-ci-e2e-test
    resourceVersion: "21831"
    uid: 37dcad3e-dd4b-4bed-b8b3-46cf0b924b0a
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e9499196b5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "21833"
    uid: 377fb802-a601-47ca-a038-900a89a43418
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e949bbc7249
    namespace: kserve-ci-e2e-test
    resourceVersion: "21834"
    uid: 36b35e09-a717-40bf-b9c2-0bb949586363
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e949c2b7f9e
    namespace: kserve-ci-e2e-test
    resourceVersion: "21835"
    uid: 1ab8bba3-956e-42fd-8234-459fe5634e3b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:41Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:41Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:41Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e959827b218
    namespace: kserve-ci-e2e-test
    resourceVersion: "21887"
    uid: ae6dc385-1621-4ab5-9f4c-9732cfabe388
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e959b2c19a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "21890"
    uid: 8fb302f2-71ba-4730-8ce9-9cc4cfadc2a0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e959ba0f98d
    namespace: kserve-ci-e2e-test
    resourceVersion: "21892"
    uid: d10f1d51-85c4-4e71-a132-f328f6c3e436
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e959bab33f2
    namespace: kserve-ci-e2e-test
    resourceVersion: "21893"
    uid: efd70f5d-5128-4bdf-a063-d292bcc1ed6d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e959ec5a095
    namespace: kserve-ci-e2e-test
    resourceVersion: "21894"
    uid: 9d32b07d-49a4-4fad-bde2-f7a7beb1c480
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:18:42Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:18:42Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e959f2f8641
    namespace: kserve-ci-e2e-test
    resourceVersion: "21895"
    uid: faf47e4e-12c3-4aeb-bb95-32a9d345680e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:21:38Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:18:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:38Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62e95d4539902
    namespace: kserve-ci-e2e-test
    resourceVersion: "23267"
    uid: a4a988c7-78b8-405e-848e-4c0e0bb002b0
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:21:37Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:21:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62ebe7305b014
    namespace: kserve-ci-e2e-test
    resourceVersion: "23254"
    uid: e5b08c02-3017-4b13-b256-e00a019b06c2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:21:37Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:21:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62ebe7306355a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23256"
    uid: c7179f2b-e9b9-412e-965f-8c91fdadfe4f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx
    namespace: kserve-ci-e2e-test
    resourceVersion: "21786"
    uid: 2f4536cd-45cb-405d-b09c-ae85b79dd560
  kind: Event
  lastTimestamp: "2026-09-17T18:21:38Z"
  message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp
    10.133.0.42:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:21:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:38Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx.18d62ebecf1b1fda
    namespace: kserve-ci-e2e-test
    resourceVersion: "23266"
    uid: ea389f6f-d3c6-4c0f-9b6e-e074b966018b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-6b666fbfdb
    namespace: kserve-ci-e2e-test
    resourceVersion: "21767"
    uid: 7c3ddbd1-4880-4461-9539-43b1dfb1b8c0
  kind: Event
  lastTimestamp: "2026-09-17T18:18:37Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-6b666fbfdb-xkrvx'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:37Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb.18d62e9474cb3902
    namespace: kserve-ci-e2e-test
    resourceVersion: "21785"
    uid: 22729d04-23e9-4729-845b-27c4c4d7b260
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-promote-v2-predictor-6b666fbfdb
    namespace: kserve-ci-e2e-test
    resourceVersion: "22575"
    uid: 7c3ddbd1-4880-4461-9539-43b1dfb1b8c0
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: 'Created pod: isvc-canary-promote-v2-predictor-6b666fbfdb-ldrxg'
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor-6b666fbfdb.18d62ea972f5c7be
    namespace: kserve-ci-e2e-test
    resourceVersion: "22590"
    uid: 13779df0-7741-4902-af37-4986e811eb05
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:36Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "21766"
    uid: e1fe2b13-c7a4-4868-ae08-011b5037fef0
  kind: Event
  lastTimestamp: "2026-09-17T18:18:36Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-6b666fbfdb from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:18:36Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:18:36Z"
    name: isvc-canary-promote-v2-predictor.18d62e94700e03c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "21768"
    uid: cc576583-582d-4e38-b1a8-f6dcc2dc5a02
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:07Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-promote-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "22569"
    uid: e1fe2b13-c7a4-4868-ae08-011b5037fef0
  kind: Event
  lastTimestamp: "2026-09-17T18:20:07Z"
  message: Scaled up replica set isvc-canary-promote-v2-predictor-6b666fbfdb from
    1 to 2
  metadata:
    creationTimestamp: "2026-09-17T18:20:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:20:07Z"
    name: isvc-canary-promote-v2-predictor.18d62ea96e840a39
    namespace: kserve-ci-e2e-test
    resourceVersion: "22580"
    uid: 01162ecc-e776-4a84-a46c-82b27f19b3db
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 21
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "21759"
    uid: a942c77a-c0db-481f-b213-a4a0f9f2c179
  kind: Event
  lastTimestamp: "2026-09-17T18:20:00Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:20:00Z"
    name: isvc-canary-promote.18d62e9471c5b400
    namespace: kserve-ci-e2e-test
    resourceVersion: "22519"
    uid: 256d45c6-fb1f-462d-aaac-2bb6e7af9c5f
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "21759"
    uid: a942c77a-c0db-481f-b213-a4a0f9f2c179
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: 'Failed to update status for InferenceService "isvc-canary-promote": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-promote":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote.18d62e94722ba887
    namespace: kserve-ci-e2e-test
    resourceVersion: "22490"
    uid: 4edca5e8-8102-4a75-8796-7c263e51a695
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:18:37Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "21759"
    uid: a942c77a-c0db-481f-b213-a4a0f9f2c179
  kind: Event
  lastTimestamp: "2026-09-17T18:19:59Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-promote": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:18:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:19:59Z"
    name: isvc-canary-promote.18d62e94722beea9
    namespace: kserve-ci-e2e-test
    resourceVersion: "22491"
    uid: c356a2a2-51ca-44f7-bf0e-50f6b69dbff2
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:20:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-promote
    namespace: kserve-ci-e2e-test
    resourceVersion: "22516"
    uid: a942c77a-c0db-481f-b213-a4a0f9f2c179
  kind: Event
  lastTimestamp: "2026-09-17T18:21:29Z"
  message: InferenceService [isvc-canary-promote] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:20:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:21:29Z"
    name: isvc-canary-promote.18d62ea7f0f67aa1
    namespace: kserve-ci-e2e-test
    resourceVersion: "23184"
    uid: 202b9abb-688e-4468-81dd-8c23e26bb2c4
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:08:15.256835Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15462"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e03ad7dff42
    namespace: kserve-ci-e2e-test
    resourceVersion: "15466"
    uid: ad742449-46f0-4a36-8ba8-6898826dddd2
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15471"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Add eth0 [10.134.0.27/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e03cf3db706
    namespace: kserve-ci-e2e-test
    resourceVersion: "15475"
    uid: 8ae54523-e151-49d2-a410-861950d258b2
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e03d2395e34
    namespace: kserve-ci-e2e-test
    resourceVersion: "15477"
    uid: 01e4a046-9c04-4c26-8d2a-3cb939e816ce
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e03d54d4939
    namespace: kserve-ci-e2e-test
    resourceVersion: "15478"
    uid: ff3f1713-de98-4589-89ef-f3a686f1650b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e03d5b55ebb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15479"
    uid: df8b9418-bf2c-4707-88c0-19ab7d6a36e7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:19Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:19Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e04c3c6320e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15517"
    uid: 4b20746f-65e2-4ba0-ac7f-70acb010c10c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:19Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:08:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:19Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e04c74e2f32
    namespace: kserve-ci-e2e-test
    resourceVersion: "15523"
    uid: ac3957a5-3e91-4adc-b822-176d3e96ab34
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:19Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:08:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:19Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e04c7d7cab1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15524"
    uid: 2e39fc34-0bb3-4c4f-935c-5432ed3bde51
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:19Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:19Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:19Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:19Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e04c7e223e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15525"
    uid: 5c87d39d-5e14-4566-8a34-a41e8eb61f4f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e04ca9e7acb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15526"
    uid: df365d84-5839-49b2-9b46-bba14dcf58e9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e04cb00a152
    namespace: kserve-ci-e2e-test
    resourceVersion: "15527"
    uid: 7960a479-e571-40e9-b560-42efa267a2b7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:09:47Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:47Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e053c674f68
    namespace: kserve-ci-e2e-test
    resourceVersion: "16374"
    uid: e2a5a64d-4b2e-4bd7-af97-53c7a051a096
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:09:45Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:09:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:45Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e18a68c7768
    namespace: kserve-ci-e2e-test
    resourceVersion: "16352"
    uid: 16fa7c8a-2183-4bd1-9d17-a9b4ee81505e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:09:45Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:09:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:45Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e18a68d0623
    namespace: kserve-ci-e2e-test
    resourceVersion: "16354"
    uid: d0a1df59-2a8c-4b52-90cf-b222157770b1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv
    namespace: kserve-ci-e2e-test
    resourceVersion: "15464"
    uid: 2a5b7de5-725a-4768-9c72-2e7d04e033c0
  kind: Event
  lastTimestamp: "2026-09-17T18:09:47Z"
  message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp
    10.134.0.27:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:09:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:47Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv.18d62e1942a7c074
    namespace: kserve-ci-e2e-test
    resourceVersion: "16373"
    uid: edbc861f-a74f-4c4f-95d2-acc837c793f6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:06:25.655170Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14452"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62dea28b9340c
    namespace: kserve-ci-e2e-test
    resourceVersion: "14459"
    uid: a7dbe153-a91b-40ed-a220-75b672d74b25
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14464"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Add eth0 [10.133.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62dea4aabc3cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "14497"
    uid: 869d0c46-95e9-4b7c-9c09-ffa3ac69880e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62dea4cdaeece
    namespace: kserve-ci-e2e-test
    resourceVersion: "14499"
    uid: f0c582cd-ae23-43d3-9ba1-6c07bf46c963
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:29Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    in 3.259s (3.259s including waiting). Image size: 308538792 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:29Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62deb0f26c9e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14540"
    uid: 676bffb2-e56c-4a6b-b51a-61172fef322e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:29Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:06:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:29Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62deb122cf3ae
    namespace: kserve-ci-e2e-test
    resourceVersion: "14542"
    uid: 221f9e2d-548a-4f20-888f-a832ffccd3d0
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:29Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:06:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:29Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62deb12906a6a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14544"
    uid: 1f6ceb88-aa41-4715-8d35-9033b72585e5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:34Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
  metadata:
    creationTimestamp: "2026-09-17T18:06:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:34Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62dec2d8b7574
    namespace: kserve-ci-e2e-test
    resourceVersion: "14601"
    uid: 11876ca3-269d-4f0c-8b35-53ea528a2e48
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:48Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    in 14.666s (14.666s including waiting). Image size: 1583073169 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:48Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62def97c0c5ed
    namespace: kserve-ci-e2e-test
    resourceVersion: "14716"
    uid: 9ad77e93-2665-40f3-9399-37748933e37a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62def9b3de13b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14718"
    uid: a696655d-9ad4-4bf5-9dab-59ef0abefc91
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62def9ba55585
    namespace: kserve-ci-e2e-test
    resourceVersion: "14721"
    uid: 045b5a39-a06b-4f9e-94e6-5624dbc1a2ff
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62def9bb3458e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14722"
    uid: b8dcb0c9-2946-44e4-8b6a-4d06ff0fc52d
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.414s (2.414s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62df02b995aff
    namespace: kserve-ci-e2e-test
    resourceVersion: "14748"
    uid: 55980cf9-b80a-47ba-bf49-69ed919602cc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62df02ee18686
    namespace: kserve-ci-e2e-test
    resourceVersion: "14750"
    uid: 6e660501-cc52-432a-b31a-31faf9c53174
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62df02f433f5e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14752"
    uid: e39b5b49-b159-44c4-8d2b-c96db446fdac
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:48Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:06:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:48Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62df066ae0af7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16379"
    uid: e3d33a76-7793-4907-a60f-5f3d022079d7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:45Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:09:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:45Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62e18a6911843
    namespace: kserve-ci-e2e-test
    resourceVersion: "16353"
    uid: c9fd84ef-42d3-449c-b120-b6ac650d0a5c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:45Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:45Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:09:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:45Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62e18a691984e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16355"
    uid: dc3902f8-de76-46ba-9515-3a24273400f1
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6
    namespace: kserve-ci-e2e-test
    resourceVersion: "14456"
    uid: a5218ca2-26c0-4458-958c-bd90d502995e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:48Z"
  message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp
    10.133.0.32:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:09:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:48Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6.18d62e19617cde61
    namespace: kserve-ci-e2e-test
    resourceVersion: "16378"
    uid: 0034b5e8-a005-4519-a0a0-32691f305244
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:06:25.574810Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14432"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62dea23ef2193
    namespace: kserve-ci-e2e-test
    resourceVersion: "14438"
    uid: 7bdc853d-ec7c-42a7-aeaf-96a98702c2fd
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-rollback-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62dea2bb6e41b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14465"
    uid: ceb0ad8c-59cb-4f71-9e1c-5101a79f513d
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14446"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Add eth0 [10.134.0.26/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62dea69c48177
    namespace: kserve-ci-e2e-test
    resourceVersion: "14511"
    uid: de34eb22-6b0e-4c89-b987-57821bfe49ee
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62dea6b82a587
    namespace: kserve-ci-e2e-test
    resourceVersion: "14514"
    uid: cd03e798-1a01-449e-8a8f-77b802e2b048
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62dea6e9f9321
    namespace: kserve-ci-e2e-test
    resourceVersion: "14515"
    uid: 85b836d3-1ba0-4e10-8e7a-d1cae95491bb
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62dea6f267aee
    namespace: kserve-ci-e2e-test
    resourceVersion: "14516"
    uid: 993ff76f-8a8b-498f-acc1-6270faba2631
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:31Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
  metadata:
    creationTimestamp: "2026-09-17T18:06:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:31Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62deb807f95dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "14569"
    uid: 9af1ef8f-75b1-4fa3-bd94-7c50871aa9a5
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    in 15.903s (15.903s including waiting). Image size: 1583073169 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62def346da73e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14701"
    uid: 3e77fc9d-0eac-4505-98ec-8767c2b46582
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62def37b3054a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14702"
    uid: 56b46dfc-1497-44f3-8501-237567e7cf4b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62def38397d82
    namespace: kserve-ci-e2e-test
    resourceVersion: "14703"
    uid: aa3eb58f-3d0c-42c3-8302-095210c44ad1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62def3846d5de
    namespace: kserve-ci-e2e-test
    resourceVersion: "14704"
    uid: e9cc7ef1-9104-42d6-ab68-ff01cccbaece
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.345s (2.345s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62defc40ffef7
    namespace: kserve-ci-e2e-test
    resourceVersion: "14728"
    uid: df57bb4d-f225-4589-b714-cd7811f8622e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62defc6dd8029
    namespace: kserve-ci-e2e-test
    resourceVersion: "14730"
    uid: 0b9bfdd7-f482-4b33-8ed9-80550eb74417
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62defc757b9e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "14732"
    uid: e56f0dc9-e0c7-4313-9f0d-7739bb1d40d5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:50Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:07:56Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:06:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:07:56Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62deff47bfac9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15256"
    uid: b672e005-8ee1-4581-ac40-712ca42d999b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:08:01Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:08:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:01Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62e006b501de2
    namespace: kserve-ci-e2e-test
    resourceVersion: "15309"
    uid: 0be45152-7b0a-47a1-ba7c-51d74585f3c9
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:08:01Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:08:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:01Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62e006b5111f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15313"
    uid: 95b651c5-d2bf-4fdd-89b2-9292285cd9af
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl
    namespace: kserve-ci-e2e-test
    resourceVersion: "14434"
    uid: 00b3ff27-6d27-4b65-bdd5-a3b3abe63dad
  kind: Event
  lastTimestamp: "2026-09-17T18:08:01Z"
  message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp
    10.134.0.26:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:08:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:01Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl.18d62e007c9845ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "15325"
    uid: 110d776c-f466-48af-8061-fa518d327de3
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-7df7cf66bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "14419"
    uid: dcbb0255-95fb-48d3-b9c2-be8cb1126e5c
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: 'Created pod: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf.18d62dea23375ce8
    namespace: kserve-ci-e2e-test
    resourceVersion: "14436"
    uid: 0c6924a6-fc5a-48e4-8594-2b8a4fdd1c2b
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-7df7cf66bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "14419"
    uid: dcbb0255-95fb-48d3-b9c2-be8cb1126e5c
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: 'Created pod: isvc-canary-rollback-predictor-7df7cf66bf-qr8p6'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf.18d62dea28031583
    namespace: kserve-ci-e2e-test
    resourceVersion: "14455"
    uid: d378ad1f-b1db-43e9-ac22-713ef98cfd1a
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-7df7cf66bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15304"
    uid: dcbb0255-95fb-48d3-b9c2-be8cb1126e5c
  kind: Event
  lastTimestamp: "2026-09-17T18:08:01Z"
  message: 'Deleted pod: isvc-canary-rollback-predictor-7df7cf66bf-sb9nl'
  metadata:
    creationTimestamp: "2026-09-17T18:08:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:01Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf.18d62e006b5b6535
    namespace: kserve-ci-e2e-test
    resourceVersion: "15310"
    uid: 1d970a5a-e887-4426-b011-54b22983eed7
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-predictor-7df7cf66bf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15447"
    uid: dcbb0255-95fb-48d3-b9c2-be8cb1126e5c
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: 'Created pod: isvc-canary-rollback-predictor-7df7cf66bf-nrzjv'
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor-7df7cf66bf.18d62e03ad061a2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15463"
    uid: e56db8a6-a631-474f-8d6c-941a59453239
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14415"
    uid: 56ecef36-e948-4375-8453-8b4d90969af7
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-7df7cf66bf from 0
    to 2
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-predictor.18d62dea1e3d8975
    namespace: kserve-ci-e2e-test
    resourceVersion: "14422"
    uid: eef6dd3f-6932-4d44-b5ca-91fdd71fdaf8
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15300"
    uid: 56ecef36-e948-4375-8453-8b4d90969af7
  kind: Event
  lastTimestamp: "2026-09-17T18:08:01Z"
  message: Scaled down replica set isvc-canary-rollback-predictor-7df7cf66bf from
    2 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:08:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:01Z"
    name: isvc-canary-rollback-predictor.18d62e006a9bff76
    namespace: kserve-ci-e2e-test
    resourceVersion: "15306"
    uid: d45317db-135f-47bf-93e4-3daaf23845fa
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15440"
    uid: 56ecef36-e948-4375-8453-8b4d90969af7
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Scaled up replica set isvc-canary-rollback-predictor-7df7cf66bf from 1
    to 2
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-predictor.18d62e03a8461d20
    namespace: kserve-ci-e2e-test
    resourceVersion: "15451"
    uid: 0ac5b875-394e-4b4c-a96c-b1929391a56d
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:06:25.483274Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14414"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62dea1e7b0612
    namespace: kserve-ci-e2e-test
    resourceVersion: "14426"
    uid: d33f5728-dbfb-4947-a604-f109a0d9aa1b
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14430"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Add eth0 [10.134.0.25/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62dea40d108fd
    namespace: kserve-ci-e2e-test
    resourceVersion: "14490"
    uid: 892967ed-b3d2-4733-8e49-4d4b83356c93
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62dea42f9f5d9
    namespace: kserve-ci-e2e-test
    resourceVersion: "14493"
    uid: a75f2519-d428-46d2-b0d5-29048453ddba
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62dea45ff0b07
    namespace: kserve-ci-e2e-test
    resourceVersion: "14494"
    uid: 89b5e95f-c247-435d-95aa-31198a9fe2aa
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62dea4671ffd1
    namespace: kserve-ci-e2e-test
    resourceVersion: "14495"
    uid: 942544b5-2ab4-4039-9b53-57488e97b2ae
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:30Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
  metadata:
    creationTimestamp: "2026-09-17T18:06:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:30Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62deb4486a27c
    namespace: kserve-ci-e2e-test
    resourceVersion: "14560"
    uid: 3ab73c36-08e8-40cf-8ff5-cb80074be9c2
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    in 16.735s (16.735s including waiting). Image size: 1583073169 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62def2a075035
    namespace: kserve-ci-e2e-test
    resourceVersion: "14695"
    uid: f7476f81-56f5-432e-9930-b0ef0c8abd7b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62def31038b8b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14697"
    uid: 2af3a86e-c622-4ab4-88b8-7a6ead9ec648
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62def324f6b5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "14699"
    uid: 9421aeb6-6d08-4421-8f4b-46568fe0fc0b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:47Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T18:06:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:47Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62def325c2884
    namespace: kserve-ci-e2e-test
    resourceVersion: "14700"
    uid: cfe8f3df-811a-427e-a520-f7f01d1d37ec
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.435s (2.435s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62defc38e787a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14727"
    uid: 379895ae-5355-4417-8782-d16fb42d8767
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62defc6ce89fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "14729"
    uid: 854725ae-78b0-4a9e-923d-e0c56aa0f44d
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62defc73311b0
    namespace: kserve-ci-e2e-test
    resourceVersion: "14731"
    uid: dc8ebb2e-03a7-4e40-8641-9f93683ce0c8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:07:57Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:07:57Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62df0304fdfa6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15269"
    uid: 082e1c00-6c85-4719-bf60-c04300b1d36d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62e03a9a3cc0e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15456"
    uid: df83a1b6-8d71-41dc-affc-bcabeb13b578
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:08:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62e03a9a46f68
    namespace: kserve-ci-e2e-test
    resourceVersion: "15457"
    uid: 07f00b01-e738-4e55-bad1-e71a30db3e06
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:17Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j
    namespace: kserve-ci-e2e-test
    resourceVersion: "14418"
    uid: 1b5ad9a2-ef90-452f-86ec-bf4e2a80e1b8
  kind: Event
  lastTimestamp: "2026-09-17T18:08:17Z"
  message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp
    10.134.0.25:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:08:17Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:17Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j.18d62e043680eb8b
    namespace: kserve-ci-e2e-test
    resourceVersion: "15498"
    uid: b2f90dd4-be80-47e0-8eb0-e974e7eac141
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-rollback-v2-predictor-65f95c5784
    namespace: kserve-ci-e2e-test
    resourceVersion: "14406"
    uid: 915a9a92-7f90-43d1-82e8-f8246b9bdc9b
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: 'Created pod: isvc-canary-rollback-v2-predictor-65f95c5784-cch4j'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-v2-predictor-65f95c5784.18d62dea1d20ea0f
    namespace: kserve-ci-e2e-test
    resourceVersion: "14417"
    uid: 990e090f-6331-46ba-93ea-57c555d76ff5
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-rollback-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14403"
    uid: 53e12dfb-48bf-4cc0-a71d-9f635536400c
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: Scaled up replica set isvc-canary-rollback-v2-predictor-65f95c5784 from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-canary-rollback-v2-predictor.18d62dea1432dc44
    namespace: kserve-ci-e2e-test
    resourceVersion: "14408"
    uid: b286618a-9560-42d8-bf0c-c701fbad1a92
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 19
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "14393"
    uid: f2353f63-78fd-4cca-836f-bf2675f8c7b2
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback.18d62dea1dc2766c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15459"
    uid: dbe965a2-61ca-44c4-acda-59cbe0e88ae9
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "14393"
    uid: f2353f63-78fd-4cca-836f-bf2675f8c7b2
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: 'Failed to update status for InferenceService "isvc-canary-rollback": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-rollback":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback.18d62dea1ea526ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "15460"
    uid: b3455c06-4ea5-4035-8e42-1ec65f00eef7
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "14393"
    uid: f2353f63-78fd-4cca-836f-bf2675f8c7b2
  kind: Event
  lastTimestamp: "2026-09-17T18:08:15Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-rollback": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:08:15Z"
    name: isvc-canary-rollback.18d62dea1ea576c1
    namespace: kserve-ci-e2e-test
    resourceVersion: "15461"
    uid: 045a55cf-31d5-4df6-a64b-1a78940a05ac
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:08Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-rollback
    namespace: kserve-ci-e2e-test
    resourceVersion: "15394"
    uid: f2353f63-78fd-4cca-836f-bf2675f8c7b2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:38Z"
  message: InferenceService [isvc-canary-rollback] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:08:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:09:38Z"
    name: isvc-canary-rollback.18d62e021fcba238
    namespace: kserve-ci-e2e-test
    resourceVersion: "16271"
    uid: 97cc1905-6def-4037-9d79-0ba28282f67a
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:09:50.587945Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16447"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-b84cdfc7-b47j6
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e19dfab8a9d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16452"
    uid: f07ce12c-e469-415b-bd9d-bf8448d52337
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e19ebd045d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16467"
    uid: 9f11cced-f1d4-4f0e-9b41-1452ad058ff2
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16458"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Add eth0 [10.133.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1a252f7ac3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16489"
    uid: 0cb51ca8-499d-426c-bba9-38ed80a9e87e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1a26da5b67
    namespace: kserve-ci-e2e-test
    resourceVersion: "16491"
    uid: 049724b1-e81a-47c6-bff5-df35b3d70a58
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1a2a5d2d2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16494"
    uid: 56640873-5d5d-4c76-b03c-47be67404ea8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1a2abe4198
    namespace: kserve-ci-e2e-test
    resourceVersion: "16495"
    uid: cf35b0c8-6a00-439b-b0ef-ab824ebf6bdd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b37883365
    namespace: kserve-ci-e2e-test
    resourceVersion: "16543"
    uid: 75d11ca3-d973-4d1e-963b-cbc06aa20db5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b3ba6e471
    namespace: kserve-ci-e2e-test
    resourceVersion: "16551"
    uid: 0244ba39-00d6-4971-8265-4622966bd2fa
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b3c0bb193
    namespace: kserve-ci-e2e-test
    resourceVersion: "16555"
    uid: a3cb77e3-589e-4768-8034-8d7f2e83029e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b3c15c63c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16557"
    uid: b7af1803-4c53-41c2-a562-09dd112e24c9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b3fae6c5d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16564"
    uid: 45e04589-b785-42c0-a845-90f9640ad766
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b405e4a49
    namespace: kserve-ci-e2e-test
    resourceVersion: "16566"
    uid: 35b588fc-b6c5-460d-8402-0b0002263d87
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:57Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:11:10Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:09:57Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:10Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e1b73e47f1f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17371"
    uid: b65b56eb-e458-442e-ba00-13c0d254487e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:11:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e2f51e30f04
    namespace: kserve-ci-e2e-test
    resourceVersion: "17497"
    uid: 0c79f3f7-9b16-418d-82f2-0bcb39cc6080
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:22Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:11:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e2f51e3db55
    namespace: kserve-ci-e2e-test
    resourceVersion: "17498"
    uid: 265807a3-1874-4e04-9c0e-89ec8b1ffe03
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:23Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16449"
    uid: 89c63f14-ac92-4725-a4fe-3cbaaea071d2
  kind: Event
  lastTimestamp: "2026-09-17T18:11:23Z"
  message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp
    10.133.0.35:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:11:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:23Z"
    name: isvc-canary-stop-predictor-b84cdfc7-b47j6.18d62e2f7a2a59a8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17513"
    uid: 9baf866e-c2d4-47ad-acac-db6d8c4e8fe5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:09:50.542225Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16437"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-predictor-b84cdfc7-c8kb8
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e19dcf19b25
    namespace: kserve-ci-e2e-test
    resourceVersion: "16440"
    uid: 4c05b2c0-16ae-426a-86b0-bdab0ed00538
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e19e48d6409
    namespace: kserve-ci-e2e-test
    resourceVersion: "16461"
    uid: 3ab24c9c-1bb1-4e72-8a83-a345526c51a3
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16445"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Add eth0 [10.134.0.30/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1a22ac90a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16483"
    uid: efdf794e-05ed-4642-b4de-6770682a63bc
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1a25178493
    namespace: kserve-ci-e2e-test
    resourceVersion: "16487"
    uid: 6517bd6e-130d-4fb2-b468-cf88b7cce6f1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1a27eb550c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16492"
    uid: 7e1e417a-5ee9-45cc-9fac-56eccfbd5822
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1a286f7f6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "16493"
    uid: b1149767-8345-49a2-80ef-2062b24d36e3
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1b3a032da3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16549"
    uid: 93ed0381-e37d-48ef-972b-a22a5170da4e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1b3e343872
    namespace: kserve-ci-e2e-test
    resourceVersion: "16559"
    uid: 1b06678c-7a36-4a28-bbcf-85889e31f18a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1b3e96afa3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16560"
    uid: b2bdcafa-e6b6-4889-9388-aed1c8d55587
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1b3ea091fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16561"
    uid: 9fc1c21d-0d5a-4bd9-8af8-77723d7fba04
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1b41777d5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16568"
    uid: 73964b04-9e15-4f9a-94c4-d8b4f092c1be
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1b41dded16
    namespace: kserve-ci-e2e-test
    resourceVersion: "16569"
    uid: 121d0148-fa31-408e-add0-1e15ff3596b0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:11:34Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:09:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:34Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e1bb1e6d52f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17629"
    uid: dad2b593-c236-46aa-a6dd-07cdfa80d544
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:11:30Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:11:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:30Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e3125da77ba
    namespace: kserve-ci-e2e-test
    resourceVersion: "17596"
    uid: c614a583-8cfb-4da8-8f17-5d369555ef9e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:11:30Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:11:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:30Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e3125db087f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17597"
    uid: 1bb1cebd-3c71-438a-a3a9-a052142050ac
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16439"
    uid: 4edabf76-33a1-495b-a1a7-c5c36c0b3929
  kind: Event
  lastTimestamp: "2026-09-17T18:11:34Z"
  message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp
    10.134.0.30:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:11:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:34Z"
    name: isvc-canary-stop-predictor-b84cdfc7-c8kb8.18d62e320c414f9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17628"
    uid: 6effb3ff-bfb7-4d03-9c04-f6ddd75c0fb5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-b84cdfc7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16419"
    uid: 5dd45435-70f0-4a23-a33d-772c59ab7625
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: 'Created pod: isvc-canary-stop-predictor-b84cdfc7-c8kb8'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor-b84cdfc7.18d62e19dc7939ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "16438"
    uid: 3915683c-a28e-486d-825e-486f4e6ee6af
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-b84cdfc7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16419"
    uid: 5dd45435-70f0-4a23-a33d-772c59ab7625
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: 'Created pod: isvc-canary-stop-predictor-b84cdfc7-b47j6'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor-b84cdfc7.18d62e19df49f3e8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16450"
    uid: 4a12281a-3c2f-4047-ab90-a4fd3c601f7f
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:22Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-predictor-b84cdfc7
    namespace: kserve-ci-e2e-test
    resourceVersion: "17489"
    uid: 5dd45435-70f0-4a23-a33d-772c59ab7625
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: 'Deleted pod: isvc-canary-stop-predictor-b84cdfc7-b47j6'
  metadata:
    creationTimestamp: "2026-09-17T18:11:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop-predictor-b84cdfc7.18d62e2f51b10f80
    namespace: kserve-ci-e2e-test
    resourceVersion: "17492"
    uid: 344ad531-c076-4868-b3de-208d6bad6c0b
  reason: SuccessfulDelete
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16417"
    uid: 5cb67ca7-b172-489f-9b78-43faee342965
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: Scaled up replica set isvc-canary-stop-predictor-b84cdfc7 from 0 to 2
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-predictor.18d62e19d8f00a8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16423"
    uid: 8bf734a1-0424-45c8-b544-cefde9dd572b
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:22Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17485"
    uid: 5cb67ca7-b172-489f-9b78-43faee342965
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: Scaled down replica set isvc-canary-stop-predictor-b84cdfc7 from 2 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:11:22Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop-predictor.18d62e2f51469ba2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17490"
    uid: 41cc15b6-9d5a-4118-9d56-698b2670835d
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:09:50.507416Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16428"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e19dade75ff
    namespace: kserve-ci-e2e-test
    resourceVersion: "16433"
    uid: 1a739cd1-94e8-432f-9a21-e584d151f61e
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-canary-stop-v2-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e19e5c790e7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16462"
    uid: af04ced2-6120-44f1-b5e4-a266b23dafd5
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16444"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Add eth0 [10.133.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1a2055aafa
    namespace: kserve-ci-e2e-test
    resourceVersion: "16480"
    uid: df6bb520-5636-4010-9152-0d61abe75c68
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1a21d7aabc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16482"
    uid: 7a4a88b1-c5af-4b1d-911d-359f5c6976c2
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1a2487c94c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16485"
    uid: aebac0a5-988d-4802-b708-6ab4aefa3570
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:51Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:09:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:51Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1a24ea5727
    namespace: kserve-ci-e2e-test
    resourceVersion: "16488"
    uid: a0ad24a2-d0b7-48b6-984f-3686dcfa42c8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1b379ebd72
    namespace: kserve-ci-e2e-test
    resourceVersion: "16544"
    uid: d056ea26-8244-4387-b06a-56517ec39967
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1b3bb276e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16554"
    uid: 8ef2f506-4ec5-42ad-9206-ed57c83082ff
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1b3c1472ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "16556"
    uid: 77f36410-dcfb-46a9-9ed1-c4aa87a381d7
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1b3c1d8fcc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16558"
    uid: f61d3a64-420c-49d7-91ff-2d19ead55e9c
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1b3fb7fc37
    namespace: kserve-ci-e2e-test
    resourceVersion: "16565"
    uid: 38face91-66c5-4da2-a1cf-83742bb748f9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:09:56Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:09:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:56Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1b405e7ce2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16567"
    uid: 1327b66c-2a97-4812-9283-6f538fcb9504
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:58Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:11:32Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:09:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:32Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e1bafcf5b55
    namespace: kserve-ci-e2e-test
    resourceVersion: "17612"
    uid: dff85e1a-8211-4eb9-9cae-c785c16354cf
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:11:30Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:11:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:30Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e3123e14cc9
    namespace: kserve-ci-e2e-test
    resourceVersion: "17589"
    uid: 853bd566-a50b-4f20-ac2f-0b91afad08e7
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:30Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:11:30Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:11:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:30Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e3123e229c8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17592"
    uid: 0d0ded6e-1b92-4377-9734-e35d0da9a49f
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:34Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16430"
    uid: 9d63981d-842e-4d90-9564-c44236b6566d
  kind: Event
  lastTimestamp: "2026-09-17T18:11:34Z"
  message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp
    10.133.0.34:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:11:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:34Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f.18d62e320a1efd3a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17627"
    uid: 4afea882-36d2-4b8d-8695-5207da7df2c6
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-canary-stop-v2-predictor-867f674bf6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16410"
    uid: c6aa895b-b12b-42df-bdc9-c0e987385a75
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: 'Created pod: isvc-canary-stop-v2-predictor-867f674bf6-wfs4f'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-v2-predictor-867f674bf6.18d62e19da4d66d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "16431"
    uid: 51a564f7-a04b-44c9-ad90-d6fa3ad759e6
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-canary-stop-v2-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16409"
    uid: 0462ae42-77db-447e-8ad1-89c758e335cc
  kind: Event
  lastTimestamp: "2026-09-17T18:09:50Z"
  message: Scaled up replica set isvc-canary-stop-v2-predictor-867f674bf6 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:50Z"
    name: isvc-canary-stop-v2-predictor.18d62e19d714a1ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "16411"
    uid: 776f062b-dcaa-49fd-a72d-24fb72a72cbf
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 21
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "16402"
    uid: eea766ce-4178-48f2-8702-9c4cc4c6fb22
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop.18d62e19d885a3e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17506"
    uid: 52623016-71f4-4d29-9f6f-04ab6dfb8d9b
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "16402"
    uid: eea766ce-4178-48f2-8702-9c4cc4c6fb22
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: 'Failed to update status for InferenceService "isvc-canary-stop": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-canary-stop":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop.18d62e19d8f7e139
    namespace: kserve-ci-e2e-test
    resourceVersion: "17503"
    uid: 65aa9c4e-700b-46e9-9fc0-2d53baf854e7
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:50Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "16402"
    uid: eea766ce-4178-48f2-8702-9c4cc4c6fb22
  kind: Event
  lastTimestamp: "2026-09-17T18:11:22Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-canary-stop": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:09:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:22Z"
    name: isvc-canary-stop.18d62e19d8f82f3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17504"
    uid: c81f6ed8-5a60-4e58-bf1b-7562f533b086
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-canary-stop
    namespace: kserve-ci-e2e-test
    resourceVersion: "17463"
    uid: eea766ce-4178-48f2-8702-9c4cc4c6fb22
  kind: Event
  lastTimestamp: "2026-09-17T18:11:20Z"
  message: InferenceService [isvc-canary-stop] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:11:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:20Z"
    name: isvc-canary-stop.18d62e2ed9c09932
    namespace: kserve-ci-e2e-test
    resourceVersion: "17464"
    uid: 2e369a27-6b3d-44a6-afdb-e1737d8d3c93
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:13:49.228492Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    namespace: kserve-ci-e2e-test
    resourceVersion: "19084"
    uid: 6d368be0-b6c1-4f16-a299-3b9db471703d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:13:49Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm.18d62e516fc108c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19090"
    uid: e9fbb38e-6d53-4178-80c6-1ef866c1341b
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    namespace: kserve-ci-e2e-test
    resourceVersion: "19094"
    uid: 6d368be0-b6c1-4f16-a299-3b9db471703d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:49Z"
  message: Add eth0 [10.134.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:13:49Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm.18d62e5191c29a47
    namespace: kserve-ci-e2e-test
    resourceVersion: "19108"
    uid: 316df260-2b2a-495f-afba-7f510aedf487
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    namespace: kserve-ci-e2e-test
    resourceVersion: "19088"
    uid: 6d368be0-b6c1-4f16-a299-3b9db471703d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:52Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:52Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm.18d62e5193428aff
    namespace: kserve-ci-e2e-test
    resourceVersion: "19135"
    uid: 19ea979f-14da-424a-a102-4919af9a05a4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    namespace: kserve-ci-e2e-test
    resourceVersion: "19088"
    uid: 6d368be0-b6c1-4f16-a299-3b9db471703d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:52Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:52Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm.18d62e5196096b28
    namespace: kserve-ci-e2e-test
    resourceVersion: "19137"
    uid: fb5fc809-6c62-46a4-8921-2528bb075074
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    namespace: kserve-ci-e2e-test
    resourceVersion: "19088"
    uid: 6d368be0-b6c1-4f16-a299-3b9db471703d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:52Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:52Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm.18d62e519680855e
    namespace: kserve-ci-e2e-test
    resourceVersion: "19140"
    uid: 90ce74e2-9aa0-40fe-904d-58ed62bc7d47
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm
    namespace: kserve-ci-e2e-test
    resourceVersion: "19088"
    uid: 6d368be0-b6c1-4f16-a299-3b9db471703d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:54Z"
  message: Stopping container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:13:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:54Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm.18d62e52a5d27084
    namespace: kserve-ci-e2e-test
    resourceVersion: "19173"
    uid: 76637c2a-8401-4aa6-8907-6af8064d7815
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19077"
    uid: 925699d7-ef0f-455f-ae16-6951caee8137
  kind: Event
  lastTimestamp: "2026-09-17T18:13:49Z"
  message: 'Created pod: isvc-init-fail-aee0cc-predictor-8588b47b8b-sp7lm'
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:49Z"
    name: isvc-init-fail-aee0cc-predictor-8588b47b8b.18d62e516f455ee8
    namespace: kserve-ci-e2e-test
    resourceVersion: "19087"
    uid: 016a15d4-1d3b-4b20-a83d-d1f1daca6435
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-init-fail-aee0cc-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19074"
    uid: c1be9c60-e2e3-4331-b600-e131218b1767
  kind: Event
  lastTimestamp: "2026-09-17T18:13:49Z"
  message: Scaled up replica set isvc-init-fail-aee0cc-predictor-8588b47b8b from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:49Z"
    name: isvc-init-fail-aee0cc-predictor.18d62e516ca53725
    namespace: kserve-ci-e2e-test
    resourceVersion: "19078"
    uid: d13f7e2a-4996-480e-905e-098c3a9fc377
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-aee0cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 2ab272ec-66f9-4cb6-9b2f-39169c2372da
  kind: Event
  lastTimestamp: "2026-09-17T18:13:52Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:52Z"
    name: isvc-init-fail-aee0cc.18d62e516ca1bc1f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19141"
    uid: 54fc8cfd-e2d9-4c2d-8ca6-85caaed899fd
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-aee0cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 2ab272ec-66f9-4cb6-9b2f-39169c2372da
  kind: Event
  lastTimestamp: "2026-09-17T18:13:49Z"
  message: 'Failed to update status for InferenceService "isvc-init-fail-aee0cc":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-aee0cc":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:49Z"
    name: isvc-init-fail-aee0cc.18d62e516d167927
    namespace: kserve-ci-e2e-test
    resourceVersion: "19081"
    uid: 25e3958b-25f6-4c54-8f9f-85313ca2b86e
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:49Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-init-fail-aee0cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19066"
    uid: 2ab272ec-66f9-4cb6-9b2f-39169c2372da
  kind: Event
  lastTimestamp: "2026-09-17T18:13:49Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-init-fail-aee0cc": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:13:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:49Z"
    name: isvc-init-fail-aee0cc.18d62e516d16b408
    namespace: kserve-ci-e2e-test
    resourceVersion: "19082"
    uid: 26a72b37-4d12-4f98-90e5-5cd7fe897538
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:22:01.845866Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23514"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:22:01Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:22:01Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec422099a79
    namespace: kserve-ci-e2e-test
    resourceVersion: "23519"
    uid: 80263440-f803-4b60-8ba2-01f4102ca295
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:02Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23523"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:02Z"
  message: Add eth0 [10.134.0.40/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:22:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:22:02Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec444105d07
    namespace: kserve-ci-e2e-test
    resourceVersion: "23532"
    uid: ab142578-9ad6-4c34-bd2d-203218b28653
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:02Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:22:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:02Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec4459216b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "23534"
    uid: a7896416-bd46-43b6-9b79-8ed77c50752d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:02Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:22:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:02Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec4484ea575
    namespace: kserve-ci-e2e-test
    resourceVersion: "23535"
    uid: fff4f2bc-7088-4823-921f-69cdc9450009
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:02Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:02Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:22:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:02Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec448c10723
    namespace: kserve-ci-e2e-test
    resourceVersion: "23536"
    uid: 9f2aba2b-abe4-4b31-a422-0de1859877dd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec57230fcf8
    namespace: kserve-ci-e2e-test
    resourceVersion: "23579"
    uid: 40816aae-44fd-4d8f-838e-62bdd1b7d6b9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec57513f0ef
    namespace: kserve-ci-e2e-test
    resourceVersion: "23581"
    uid: f7e07d3f-735b-4a1c-a61e-a5a2da75d6f5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec575899869
    namespace: kserve-ci-e2e-test
    resourceVersion: "23582"
    uid: 989bd0b2-7428-44f8-8b73-e53cb27b8010
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec5759398cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "23583"
    uid: 7de59eae-d628-40d9-9a31-34edeadeef01
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec578770a47
    namespace: kserve-ci-e2e-test
    resourceVersion: "23587"
    uid: 7f96f36b-a2f4-4841-8b69-b22175bf99c2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec578e9241b
    namespace: kserve-ci-e2e-test
    resourceVersion: "23588"
    uid: 91b81992-83b1-4e5d-9284-66ae1806722f
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:07Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4e7ded23a7086e8e0da66cca876fe26538e5d068591452860f28aa920a1e2c50"
  metadata:
    creationTimestamp: "2026-09-17T18:22:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:07Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec578f64d36
    namespace: kserve-ci-e2e-test
    resourceVersion: "23589"
    uid: 886fca29-9dff-4a3e-b60b-637f71102760
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:10Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4e7ded23a7086e8e0da66cca876fe26538e5d068591452860f28aa920a1e2c50"
    in 2.643s (2.643s including waiting). Image size: 240300108 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:22:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:10Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec616887ea8
    namespace: kserve-ci-e2e-test
    resourceVersion: "23609"
    uid: 652c58cf-6c53-4892-a0a8-b7b3c51a7ae3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:10Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-17T18:22:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:10Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec6199ba4b1
    namespace: kserve-ci-e2e-test
    resourceVersion: "23610"
    uid: 8690c814-febc-41c7-ae94-97854b0bc7be
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:22:10Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-17T18:22:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:22:10Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec61a1f3581
    namespace: kserve-ci-e2e-test
    resourceVersion: "23612"
    uid: 8dc5ad63-0a7a-4b97-bc11-99c21a70a573
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:23:43Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:22:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:43Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec6261aa922
    namespace: kserve-ci-e2e-test
    resourceVersion: "24363"
    uid: b347257f-49b1-4ffe-8227-97d5454a986d
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:23:43Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T18:22:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:43Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62ec661ee0fe2
    namespace: kserve-ci-e2e-test
    resourceVersion: "24364"
    uid: 899c6971-7a08-4376-a84d-f1d82abc00cf
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:23:37Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:23:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:37Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62eda4d135306
    namespace: kserve-ci-e2e-test
    resourceVersion: "24266"
    uid: 3d8339ad-a266-404f-a228-cb6ec4766f35
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:23:37Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-17T18:23:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:37Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62eda4d13991b
    namespace: kserve-ci-e2e-test
    resourceVersion: "24270"
    uid: 6c0b2673-38ed-4082-a7d8-8aa572a9cd35
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:23:37Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:23:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:37Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62eda4d143355
    namespace: kserve-ci-e2e-test
    resourceVersion: "24272"
    uid: 4e99d3d2-7665-4908-b143-d096ecd9a2a2
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7
    namespace: kserve-ci-e2e-test
    resourceVersion: "23517"
    uid: 071e5b27-e983-4fb7-b47d-72c409bdaaef
  kind: Event
  lastTimestamp: "2026-09-17T18:23:52Z"
  message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp
    10.134.0.40:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:23:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:52Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7.18d62eda681f9698
    namespace: kserve-ci-e2e-test
    resourceVersion: "24428"
    uid: 6313861d-b9bd-4275-813a-69020ae80405
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-cipn-predictor-6d54d95b97
    namespace: kserve-ci-e2e-test
    resourceVersion: "23503"
    uid: 7b47c9a6-8242-429f-8d9e-45e6c0672549
  kind: Event
  lastTimestamp: "2026-09-17T18:22:01Z"
  message: 'Created pod: isvc-logger-raw-cipn-predictor-6d54d95b97-vmkp7'
  metadata:
    creationTimestamp: "2026-09-17T18:22:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:22:01Z"
    name: isvc-logger-raw-cipn-predictor-6d54d95b97.18d62ec421644263
    namespace: kserve-ci-e2e-test
    resourceVersion: "23516"
    uid: 7759f549-b801-406c-9630-77da44bb43be
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:01Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23501"
    uid: 96e159ea-accf-4937-9d80-75a9371e6bee
  kind: Event
  lastTimestamp: "2026-09-17T18:22:01Z"
  message: Scaled up replica set isvc-logger-raw-cipn-predictor-6d54d95b97 from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T18:22:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:22:01Z"
    name: isvc-logger-raw-cipn-predictor.18d62ec41e713b1d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23506"
    uid: 31500e7d-9ca3-4e02-b22a-379e8990177e
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23502"
    uid: 1e41c07e-6694-4764-b7e1-59f6cf424b25
  kind: Event
  lastTimestamp: "2026-09-17T18:22:31Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:22:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:22:31Z"
    name: isvc-logger-raw-cipn-predictor.18d62ec79cdb2b78
    namespace: kserve-ci-e2e-test
    resourceVersion: "23771"
    uid: 7ffaa612-c4e1-40d8-91e5-31aa4d4e8865
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23502"
    uid: 1e41c07e-6694-4764-b7e1-59f6cf424b25
  kind: Event
  lastTimestamp: "2026-09-17T18:22:31Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:22:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:22:31Z"
    name: isvc-logger-raw-cipn-predictor.18d62ec79d44d2cc
    namespace: kserve-ci-e2e-test
    resourceVersion: "23772"
    uid: b2646dab-9e1c-4916-a86e-1eea55cec8f7
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:46Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23668"
    uid: 1e41c07e-6694-4764-b7e1-59f6cf424b25
  kind: Event
  lastTimestamp: "2026-09-17T18:23:31Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:22:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:23:31Z"
    name: isvc-logger-raw-cipn-predictor.18d62ece9b113995
    namespace: kserve-ci-e2e-test
    resourceVersion: "24206"
    uid: da0b256c-0eab-4feb-9648-852c9fb1055a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:46Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23668"
    uid: 1e41c07e-6694-4764-b7e1-59f6cf424b25
  kind: Event
  lastTimestamp: "2026-09-17T18:23:31Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:22:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:23:31Z"
    name: isvc-logger-raw-cipn-predictor.18d62ece9b8f1567
    namespace: kserve-ci-e2e-test
    resourceVersion: "24207"
    uid: b2eab5e6-9142-4022-83ed-b93475d1b246
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23495"
    uid: 180e4802-2740-4c75-90fe-d492cd196d21
  kind: Event
  lastTimestamp: "2026-09-17T18:23:23Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:22:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:23:23Z"
    name: isvc-logger-raw-cipn.18d62ec41e7306c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "24150"
    uid: 2eb3b90f-827d-4491-86d7-3c497f4112f4
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:01Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23495"
    uid: 180e4802-2740-4c75-90fe-d492cd196d21
  kind: Event
  lastTimestamp: "2026-09-17T18:22:01Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-cipn": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cipn":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:22:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:22:01Z"
    name: isvc-logger-raw-cipn.18d62ec41f034e57
    namespace: kserve-ci-e2e-test
    resourceVersion: "23509"
    uid: af5ad516-c5a8-44ca-987b-2254c70b838c
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:23Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "24148"
    uid: 180e4802-2740-4c75-90fe-d492cd196d21
  kind: Event
  lastTimestamp: "2026-09-17T18:23:23Z"
  message: InferenceService [isvc-logger-raw-cipn] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:23:23Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:23:23Z"
    name: isvc-logger-raw-cipn.18d62ed72c1552c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "24149"
    uid: 7c46d836-2a13-4bfb-8619-8372f3e1d844
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:12:31.032101Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18320"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:12:31Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:12:31Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e3f3ae30c8b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18325"
    uid: 88a09301-ddaa-4c8c-99f3-f70acd78c65d
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:31Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:31Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-fc652-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:12:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:31Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e3f43d60591
    namespace: kserve-ci-e2e-test
    resourceVersion: "18335"
    uid: e72c6622-b182-45d9-860d-7ddef6f6a008
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:32Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18330"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:32Z"
  message: Add eth0 [10.133.0.39/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:12:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:12:32Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e3f80b4cc61
    namespace: kserve-ci-e2e-test
    resourceVersion: "18344"
    uid: 2cf8ccda-4886-4c6e-a6f7-5b8e607ce62e
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:32Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:12:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:32Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e3f8254f377
    namespace: kserve-ci-e2e-test
    resourceVersion: "18346"
    uid: c0f5c587-6b21-48b2-b4e5-dc820c1691ff
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:32Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:12:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:32Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e3f85778895
    namespace: kserve-ci-e2e-test
    resourceVersion: "18347"
    uid: 95a74972-fc56-4c36-9761-7f58e1ddf1c2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:32Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:32Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:12:32Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:32Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e3f85e0001f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18348"
    uid: 020f3ce1-6754-4a16-b44a-a5be61802700
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40ac33aa70
    namespace: kserve-ci-e2e-test
    resourceVersion: "18390"
    uid: e43841fc-f442-4c67-a5b4-44752520d797
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40aeedce18
    namespace: kserve-ci-e2e-test
    resourceVersion: "18394"
    uid: 28ab2b64-16f6-4056-af06-1c60df6c437f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40af533f21
    namespace: kserve-ci-e2e-test
    resourceVersion: "18395"
    uid: be896f6a-26a8-4489-8bf7-511b0813ff0b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40af5d1a10
    namespace: kserve-ci-e2e-test
    resourceVersion: "18396"
    uid: c010e3e9-8829-460e-a2b4-39a98ef67d27
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40b2a48091
    namespace: kserve-ci-e2e-test
    resourceVersion: "18399"
    uid: 1564d0cf-6965-4cde-af9a-f17c2bc767ef
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40b30d8ef4
    namespace: kserve-ci-e2e-test
    resourceVersion: "18401"
    uid: a5f9cb8f-9ab0-48d4-a37c-837a81d45593
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: Container image "quay.io/opendatahub/kserve-agent@sha256:4e7ded23a7086e8e0da66cca876fe26538e5d068591452860f28aa920a1e2c50"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40b319a1b9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18402"
    uid: 49d7ad68-e78d-470c-8525-fa8adf07c946
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40b5db7e3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18403"
    uid: 7da3dd57-e8f7-4cf2-909b-69b15c57325f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-17T18:12:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40b642dac2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18404"
    uid: e1535107-4572-47be-89f4-8643adf0558b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:14:08Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:12:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:08Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40e7f4b635
    namespace: kserve-ci-e2e-test
    resourceVersion: "19443"
    uid: 4198547a-ae2d-4af6-9d9f-4eccce0c4af9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:14:08Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T18:12:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:08Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e40e7fda57a
    namespace: kserve-ci-e2e-test
    resourceVersion: "19442"
    uid: 6485b1f5-f119-457e-acd6-d3822c84e035
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e5571c84de7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19373"
    uid: b6d05a38-9c6a-4aa7-9305-a80163d7af1c
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e5571c87e18
    namespace: kserve-ci-e2e-test
    resourceVersion: "19375"
    uid: 7b5b6bda-e944-4ecc-85ec-dceec1adfd49
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e5571c8df40
    namespace: kserve-ci-e2e-test
    resourceVersion: "19377"
    uid: c7c7dbd6-3f50-4fcd-b0c7-196b5eced822
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:09Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18323"
    uid: e8046de1-3a6d-458f-a741-7637c42dfa48
  kind: Event
  lastTimestamp: "2026-09-17T18:14:14Z"
  message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp
    10.133.0.39:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:14:09Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:14Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd.18d62e56182de9de
    namespace: kserve-ci-e2e-test
    resourceVersion: "19495"
    uid: e9beea0c-6f96-4c7e-9f71-60d9cb57873b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:31Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf
    namespace: kserve-ci-e2e-test
    resourceVersion: "18310"
    uid: 09176747-2793-44f2-9158-f5a7aaa9c0de
  kind: Event
  lastTimestamp: "2026-09-17T18:12:31Z"
  message: 'Created pod: isvc-logger-raw-fc652-predictor-6fff76ccbf-8k5zd'
  metadata:
    creationTimestamp: "2026-09-17T18:12:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:31Z"
    name: isvc-logger-raw-fc652-predictor-6fff76ccbf.18d62e3f3a4c51b4
    namespace: kserve-ci-e2e-test
    resourceVersion: "18322"
    uid: f41fe32c-3aad-4472-9be6-1391b8babad6
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:30Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-logger-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18308"
    uid: de1346a6-4933-406d-86ff-04a8ebbf9559
  kind: Event
  lastTimestamp: "2026-09-17T18:12:30Z"
  message: Scaled up replica set isvc-logger-raw-fc652-predictor-6fff76ccbf from 0
    to 1
  metadata:
    creationTimestamp: "2026-09-17T18:12:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:30Z"
    name: isvc-logger-raw-fc652-predictor.18d62e3f37464f48
    namespace: kserve-ci-e2e-test
    resourceVersion: "18312"
    uid: 796d2627-36e5-4532-89d2-75fb651e5eb7
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:45Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18309"
    uid: 62db1f7a-e10a-4ba4-a6d9-52d0470a41d9
  kind: Event
  lastTimestamp: "2026-09-17T18:13:01Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:12:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:01Z"
    name: isvc-logger-raw-fc652-predictor.18d62e42b5e48edc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18592"
    uid: 6a743a52-9c96-490d-ad4c-f4608f89d51e
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:45Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18309"
    uid: 62db1f7a-e10a-4ba4-a6d9-52d0470a41d9
  kind: Event
  lastTimestamp: "2026-09-17T18:13:01Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:12:45Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:01Z"
    name: isvc-logger-raw-fc652-predictor.18d62e42b6471aee
    namespace: kserve-ci-e2e-test
    resourceVersion: "18593"
    uid: 65ea20ed-0bef-41f7-b83b-77781fc252e7
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18487"
    uid: 62db1f7a-e10a-4ba4-a6d9-52d0470a41d9
  kind: Event
  lastTimestamp: "2026-09-17T18:14:01Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:13:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:01Z"
    name: isvc-logger-raw-fc652-predictor.18d62e49b3e981b7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19282"
    uid: 1956bb81-119e-4ea8-9121-42a738197e8f
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:16Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-logger-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18487"
    uid: 62db1f7a-e10a-4ba4-a6d9-52d0470a41d9
  kind: Event
  lastTimestamp: "2026-09-17T18:14:01Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:13:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:01Z"
    name: isvc-logger-raw-fc652-predictor.18d62e49b44f19bc
    namespace: kserve-ci-e2e-test
    resourceVersion: "19283"
    uid: 2382a4ef-9525-4508-949b-92a226605818
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:30Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-fc652
    namespace: kserve-ci-e2e-test
    resourceVersion: "18300"
    uid: 45df1d7d-6e1a-4883-9fbd-38ae210f4ac0
  kind: Event
  lastTimestamp: "2026-09-17T18:13:58Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:12:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:58Z"
    name: isvc-logger-raw-fc652.18d62e3f3751e9eb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19218"
    uid: 0755bb84-610a-48d6-9ed3-46a9988cff50
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:30Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-fc652
    namespace: kserve-ci-e2e-test
    resourceVersion: "18300"
    uid: 45df1d7d-6e1a-4883-9fbd-38ae210f4ac0
  kind: Event
  lastTimestamp: "2026-09-17T18:12:30Z"
  message: 'Failed to update status for InferenceService "isvc-logger-raw-fc652":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-fc652":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:12:30Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:12:30Z"
    name: isvc-logger-raw-fc652.18d62e3f37e399ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "18316"
    uid: e7629694-419e-40f8-941f-eb72e3923c9a
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:58Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-logger-raw-fc652
    namespace: kserve-ci-e2e-test
    resourceVersion: "19216"
    uid: 45df1d7d-6e1a-4883-9fbd-38ae210f4ac0
  kind: Event
  lastTimestamp: "2026-09-17T18:13:58Z"
  message: InferenceService [isvc-logger-raw-fc652] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:13:58Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:58Z"
    name: isvc-logger-raw-fc652.18d62e53a38d9401
    namespace: kserve-ci-e2e-test
    resourceVersion: "19217"
    uid: 3973973e-e5dd-4720-b7b9-22f731ba3645
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:11:40.807998Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17703"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-primary-c81569-predictor-d8cc8857-btdk8
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:11:40Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e33894bbf2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17708"
    uid: 2e8cd80e-c9b6-4357-9550-49f1585303aa
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:40Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:40Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-c81569-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:40Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e33938f69f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17721"
    uid: 79e95fca-75db-4d80-ba78-4a032b4a09b5
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:41Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17713"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:41Z"
  message: Add eth0 [10.134.0.31/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:11:41Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:11:41Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e33cef3dcc6
    namespace: kserve-ci-e2e-test
    resourceVersion: "17734"
    uid: 88beb63d-db82-4e3e-8331-3d22841eea49
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:42Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:11:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:42Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e33d1058cf3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17736"
    uid: d36c74d3-17dc-45d6-8fbb-b1f4fd64a8a4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:42Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:11:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:42Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e33d4050c5a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17737"
    uid: 71237981-bf7d-435b-84c4-69ff585fd037
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:42Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:11:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:42Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e33d4759513
    namespace: kserve-ci-e2e-test
    resourceVersion: "17738"
    uid: cbb234e2-605f-4151-8e67-2cbf108557bd
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:46Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:46Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e34f366ba6f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17787"
    uid: 1dc63de4-b0e0-4e44-a97d-f3c66ddb00a0
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:46Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:46Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e34f67d1cbc
    namespace: kserve-ci-e2e-test
    resourceVersion: "17789"
    uid: f8f80083-28a0-427d-9ddb-906bb73f32c6
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:46Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:46Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e34f6f0b1c3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17792"
    uid: 778fb8bd-c43b-4f48-9063-96dcc36b9246
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:46Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:46Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e34f6fa0023
    namespace: kserve-ci-e2e-test
    resourceVersion: "17793"
    uid: 6ed56ec9-539b-4970-a25c-a04f405c59a3
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:46Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:46Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:11:46Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:46Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e34fa073a0f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17795"
    uid: 14a40cde-9350-4e00-a8e8-f24367910b80
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:11:47Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:11:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:47Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e34fa7c436e
    namespace: kserve-ci-e2e-test
    resourceVersion: "17797"
    uid: ddfba226-82b8-4e8b-b639-9108c15ded8a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:12:53Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:11:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:53Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e352f732dae
    namespace: kserve-ci-e2e-test
    resourceVersion: "18541"
    uid: 58e2314a-1948-4e11-9aa1-25ea9f3d262b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:13:44Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:13:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:44Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e5038948b79
    namespace: kserve-ci-e2e-test
    resourceVersion: "19014"
    uid: 137d8292-0d59-4c3e-b9a6-7945e81294ae
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:44Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8
    namespace: kserve-ci-e2e-test
    resourceVersion: "17705"
    uid: 2d04c05e-6542-4f77-b794-7ee1af0be89a
  kind: Event
  lastTimestamp: "2026-09-17T18:13:44Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:13:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:44Z"
    name: isvc-primary-c81569-predictor-d8cc8857-btdk8.18d62e5038951f30
    namespace: kserve-ci-e2e-test
    resourceVersion: "19015"
    uid: e3221f04-a6ac-4add-9787-b1c854a3c5fe
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:40Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-primary-c81569-predictor-d8cc8857
    namespace: kserve-ci-e2e-test
    resourceVersion: "17693"
    uid: 7beb5dd0-433e-48ea-8283-b7770a76baf6
  kind: Event
  lastTimestamp: "2026-09-17T18:11:40Z"
  message: 'Created pod: isvc-primary-c81569-predictor-d8cc8857-btdk8'
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:11:40Z"
    name: isvc-primary-c81569-predictor-d8cc8857.18d62e33886fe9a0
    namespace: kserve-ci-e2e-test
    resourceVersion: "17704"
    uid: d5403ede-c370-4a8a-93c4-7336c904b357
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:40Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-primary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17691"
    uid: 1eb6cdea-daec-49b8-9370-22cc752e3166
  kind: Event
  lastTimestamp: "2026-09-17T18:11:40Z"
  message: Scaled up replica set isvc-primary-c81569-predictor-d8cc8857 from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:11:40Z"
    name: isvc-primary-c81569-predictor.18d62e3384a1755f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17695"
    uid: d19170fd-36eb-402f-825a-fcb32a4a566f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:55Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17692"
    uid: 23c7817f-b39c-49d5-817f-55191d7471e6
  kind: Event
  lastTimestamp: "2026-09-17T18:12:25Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:11:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:25Z"
    name: isvc-primary-c81569-predictor.18d62e370314edf1
    namespace: kserve-ci-e2e-test
    resourceVersion: "18257"
    uid: 1fc4cf8b-81d5-41c8-9e1b-b4941652b5bd
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:55Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17692"
    uid: 23c7817f-b39c-49d5-817f-55191d7471e6
  kind: Event
  lastTimestamp: "2026-09-17T18:12:25Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:11:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:25Z"
    name: isvc-primary-c81569-predictor.18d62e370393646a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18258"
    uid: a7403128-034b-436f-afb8-b32255cc3b21
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17877"
    uid: 23c7817f-b39c-49d5-817f-55191d7471e6
  kind: Event
  lastTimestamp: "2026-09-17T18:13:40Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:12:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:40Z"
    name: isvc-primary-c81569-predictor.18d62e41807c07b8
    namespace: kserve-ci-e2e-test
    resourceVersion: "18959"
    uid: f62cead5-675b-484c-a7d1-55748bea6ac5
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-primary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17877"
    uid: 23c7817f-b39c-49d5-817f-55191d7471e6
  kind: Event
  lastTimestamp: "2026-09-17T18:13:40Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:12:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:40Z"
    name: isvc-primary-c81569-predictor.18d62e4180e7020d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18960"
    uid: 4c0873eb-070d-4a41-99ab-66ed9675d81a
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:40Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "17683"
    uid: 6d07f246-e7ec-4f73-86ab-1974294b12f3
  kind: Event
  lastTimestamp: "2026-09-17T18:13:04Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:04Z"
    name: isvc-primary-c81569.18d62e33849f9a2f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18626"
    uid: c6b60afd-aaa7-453c-ab0f-2cc7549e512f
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:40Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "17683"
    uid: 6d07f246-e7ec-4f73-86ab-1974294b12f3
  kind: Event
  lastTimestamp: "2026-09-17T18:11:40Z"
  message: 'Failed to update status for InferenceService "isvc-primary-c81569": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c81569":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:40Z"
    name: isvc-primary-c81569.18d62e338508f353
    namespace: kserve-ci-e2e-test
    resourceVersion: "17697"
    uid: 1b9ba3ce-45da-49d8-8184-0578620e3150
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:40Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "17683"
    uid: 6d07f246-e7ec-4f73-86ab-1974294b12f3
  kind: Event
  lastTimestamp: "2026-09-17T18:11:40Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-primary-c81569": the object has been
    modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:11:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:40Z"
    name: isvc-primary-c81569.18d62e3385094175
    namespace: kserve-ci-e2e-test
    resourceVersion: "17698"
    uid: ce99d369-c39e-4159-95ef-4398c45d21d7
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:04Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-primary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "18624"
    uid: 6d07f246-e7ec-4f73-86ab-1974294b12f3
  kind: Event
  lastTimestamp: "2026-09-17T18:13:04Z"
  message: InferenceService [isvc-primary-c81569] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:13:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:04Z"
    name: isvc-primary-c81569.18d62e46eb069f3f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18625"
    uid: 8b4078df-83bd-4858-92d9-e9459d48e11c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:06:25.743880Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14469"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62dea2e02ec68
    namespace: kserve-ci-e2e-test
    resourceVersion: "14473"
    uid: d447f2f8-3589-4e70-8397-0eb8da054b2f
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14476"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Add eth0 [10.133.0.33/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62dea4fcad497
    namespace: kserve-ci-e2e-test
    resourceVersion: "14501"
    uid: 9657861f-e3e2-4350-bd5c-d32758b32a88
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:26Z"
  message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
  metadata:
    creationTimestamp: "2026-09-17T18:06:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:26Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62dea51ec999b
    namespace: kserve-ci-e2e-test
    resourceVersion: "14503"
    uid: 9987ab7c-c66e-4847-83c8-c1120d55ea7e
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:29Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    in 3.169s (3.169s including waiting). Image size: 308538792 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:29Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62deb0edc7638
    namespace: kserve-ci-e2e-test
    resourceVersion: "14539"
    uid: 56d7251b-790d-4fdf-ac5b-f07d654c2967
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:29Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:06:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:29Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62deb1230a140
    namespace: kserve-ci-e2e-test
    resourceVersion: "14543"
    uid: 6f2c20ab-d713-48e3-8f72-2e26a4e467ec
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:29Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:06:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:29Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62deb1294f9d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "14545"
    uid: 975bd2c5-9ff4-4299-a026-38f60139a0d1
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:33Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:33Z"
  message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
  metadata:
    creationTimestamp: "2026-09-17T18:06:33Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:33Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62debf14297eb
    namespace: kserve-ci-e2e-test
    resourceVersion: "14589"
    uid: 4ab22716-9636-46f5-a1db-45908b836996
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:48Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:48Z"
  message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    in 15.665s (15.665s including waiting). Image size: 1583073169 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:48Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62def97056720
    namespace: kserve-ci-e2e-test
    resourceVersion: "14715"
    uid: 85d9e423-2a38-403e-979d-83092b9edc4f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62def9b322385
    namespace: kserve-ci-e2e-test
    resourceVersion: "14717"
    uid: be8dc45d-715f-4fd5-b722-97fe01583c4c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62def9b96e2b7
    namespace: kserve-ci-e2e-test
    resourceVersion: "14719"
    uid: 58903915-06a1-4f0d-9f2f-367b6db6d5b5
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:49Z"
  message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
  metadata:
    creationTimestamp: "2026-09-17T18:06:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:49Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62def9ba4495a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14720"
    uid: c758095c-836e-4a56-8d45-93ff8a546ab7
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    in 2.406s (2.406s including waiting). Image size: 211946088 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df02b14537e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14747"
    uid: 9c40baa8-9c94-4c07-8fc5-0799af5dff6e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df02ed83d09
    namespace: kserve-ci-e2e-test
    resourceVersion: "14749"
    uid: 1da6485a-2cd1-4d1d-9b3c-a2a7602155a3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df02f3dcd11
    namespace: kserve-ci-e2e-test
    resourceVersion: "14751"
    uid: 5451f9b1-1a90-4739-9abc-bf4f0615f2f0
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:51Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:51Z"
  message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4e7ded23a7086e8e0da66cca876fe26538e5d068591452860f28aa920a1e2c50"
  metadata:
    creationTimestamp: "2026-09-17T18:06:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:51Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df02f4bbf2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "14753"
    uid: 495dffec-cdfa-40d9-b960-d35fa5d34e5b
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:54Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4e7ded23a7086e8e0da66cca876fe26538e5d068591452860f28aa920a1e2c50"
    in 2.466s (2.466s including waiting). Image size: 240300108 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:06:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:54Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df0c24c2174
    namespace: kserve-ci-e2e-test
    resourceVersion: "14776"
    uid: f776cf8c-5d25-4920-aaf8-c6f48e1527ac
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:54Z"
  message: 'Created container: agent'
  metadata:
    creationTimestamp: "2026-09-17T18:06:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:54Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df0c4fa8182
    namespace: kserve-ci-e2e-test
    resourceVersion: "14778"
    uid: 627c0996-5729-4471-ac9a-28d241c345af
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:06:54Z"
  message: Started container agent
  metadata:
    creationTimestamp: "2026-09-17T18:06:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:06:54Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df0c55fe3f0
    namespace: kserve-ci-e2e-test
    resourceVersion: "14779"
    uid: 2571332b-5bfa-4cf8-9992-f97ff0b95c20
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:54Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:08:27Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:06:54Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:27Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df0deba2f27
    namespace: kserve-ci-e2e-test
    resourceVersion: "15722"
    uid: e603c458-fab3-442b-9438-8c5a20430dac
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:55Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:08:27Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T18:06:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:27Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62df11a965e33
    namespace: kserve-ci-e2e-test
    resourceVersion: "15723"
    uid: abf4014d-1d74-47fb-9fdc-94979f5c8019
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62e04d66f7ed8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15548"
    uid: e3ec5231-3ee0-417b-8d18-29e6c952b75e
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{agent}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: Stopping container agent
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62e04d66fd7a8
    namespace: kserve-ci-e2e-test
    resourceVersion: "15553"
    uid: 7ac547f6-57f8-4321-98b2-d5a7380b62b6
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62e04d6701aa0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15550"
    uid: 9c2d7a7a-f0f0-45ea-859a-407c9e324c33
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4
    namespace: kserve-ci-e2e-test
    resourceVersion: "14470"
    uid: 29a8f91a-95a3-4aea-b66d-9a71cfd0b5ee
  kind: Event
  lastTimestamp: "2026-09-17T18:08:36Z"
  message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp
    10.133.0.33:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:36Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4.18d62e0520dd1f28
    namespace: kserve-ci-e2e-test
    resourceVersion: "15797"
    uid: efd5ea9e-15f9-4a7c-a93c-e9b68903e005
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664
    namespace: kserve-ci-e2e-test
    resourceVersion: "14448"
    uid: 90501674-6d7d-4288-a6df-7b2678edcefe
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: 'Created pod: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664-zrvx4'
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor-8f6596664.18d62dea2d71f536
    namespace: kserve-ci-e2e-test
    resourceVersion: "14471"
    uid: bda2ab9e-c55e-4dd4-b0e1-c70539fa955d
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-raw-sklearn-batcher-cb079-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14445"
    uid: 1d3dafd2-4220-4f36-aa69-4f2241effa92
  kind: Event
  lastTimestamp: "2026-09-17T18:06:25Z"
  message: Scaled up replica set isvc-raw-sklearn-batcher-cb079-predictor-8f6596664
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:06:25Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor.18d62dea2732dd0e
    namespace: kserve-ci-e2e-test
    resourceVersion: "14451"
    uid: 66584e90-9fa5-410c-8eb7-867b8869d650
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-cb079-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14447"
    uid: 4288f9c5-f542-407b-ad79-7d664927bdad
  kind: Event
  lastTimestamp: "2026-09-17T18:07:25Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:06:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:07:25Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor.18d62deda72e5755
    namespace: kserve-ci-e2e-test
    resourceVersion: "15023"
    uid: b33e83d0-bfdb-4387-a1c7-56a524f1edd7
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-cb079-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14447"
    uid: 4288f9c5-f542-407b-ad79-7d664927bdad
  kind: Event
  lastTimestamp: "2026-09-17T18:07:25Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:06:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:07:25Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor.18d62deda7c2af8c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15024"
    uid: d722bdf6-27ed-4dbd-90f0-5de867ce779c
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:07:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-cb079-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14652"
    uid: 4288f9c5-f542-407b-ad79-7d664927bdad
  kind: Event
  lastTimestamp: "2026-09-17T18:08:10Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:07:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:10Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor.18d62dfba5e58000
    namespace: kserve-ci-e2e-test
    resourceVersion: "15412"
    uid: 9ac86207-0583-46d0-a601-1652beba6a66
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:07:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-raw-sklearn-batcher-cb079-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "14652"
    uid: 4288f9c5-f542-407b-ad79-7d664927bdad
  kind: Event
  lastTimestamp: "2026-09-17T18:08:10Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:07:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:10Z"
    name: isvc-raw-sklearn-batcher-cb079-predictor.18d62dfba695f264
    namespace: kserve-ci-e2e-test
    resourceVersion: "15413"
    uid: 4a134259-a328-440a-9433-77a2063cd32f
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:06:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-cb079
    namespace: kserve-ci-e2e-test
    resourceVersion: "14431"
    uid: 4faa16a1-2392-41ba-99e5-8eec29d5f665
  kind: Event
  lastTimestamp: "2026-09-17T18:08:07Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:06:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:08:07Z"
    name: isvc-raw-sklearn-batcher-cb079.18d62dea26f82907
    namespace: kserve-ci-e2e-test
    resourceVersion: "15379"
    uid: dbdacd61-9316-4e2e-beee-006650795658
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:07Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-raw-sklearn-batcher-cb079
    namespace: kserve-ci-e2e-test
    resourceVersion: "15377"
    uid: 4faa16a1-2392-41ba-99e5-8eec29d5f665
  kind: Event
  lastTimestamp: "2026-09-17T18:08:07Z"
  message: InferenceService [isvc-raw-sklearn-batcher-cb079] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:08:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:08:07Z"
    name: isvc-raw-sklearn-batcher-cb079.18d62e01e5602ce4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15378"
    uid: 3585837e-19b5-4715-98c0-ee35ba234dee
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:13:25.962287Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18809"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:13:25Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:13:25Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4c04fafa48
    namespace: kserve-ci-e2e-test
    resourceVersion: "18815"
    uid: 63696180-b709-4e26-a9c3-6550ca30ebd4
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:26Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18813"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:26Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-c81569-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:13:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:26Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4c12126e1a
    namespace: kserve-ci-e2e-test
    resourceVersion: "18823"
    uid: d0c36804-e0df-4b20-9aa9-1158e440a785
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:27Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18819"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:27Z"
  message: Add eth0 [10.134.0.32/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:13:27Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4c4abe32ab
    namespace: kserve-ci-e2e-test
    resourceVersion: "18832"
    uid: 135390d4-88ec-404f-bc77-45e9bc4ffb7d
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18813"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:33Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:33Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4c4cbba0eb
    namespace: kserve-ci-e2e-test
    resourceVersion: "18887"
    uid: 9300aa31-7e53-4c7f-b3db-e867e3d9e19a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18813"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:33Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:33Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4c50fa0a5c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18890"
    uid: f1e7f2c2-6c3c-433f-b3f3-67f84872917f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18813"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:33Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:13:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:33Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4c516fb1af
    namespace: kserve-ci-e2e-test
    resourceVersion: "18892"
    uid: e4551f63-7779-46c1-b521-7caf4ee2bc2c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9
    namespace: kserve-ci-e2e-test
    resourceVersion: "18813"
    uid: ce7faf98-4168-4b83-8889-d5499f2b0a9d
  kind: Event
  lastTimestamp: "2026-09-17T18:13:40Z"
  message: Back-off restarting failed container storage-initializer in pod isvc-secondary-c81569-predictor-9b69cb5d-q5cn9_kserve-ci-e2e-test(ce7faf98-4168-4b83-8889-d5499f2b0a9d)
  metadata:
    creationTimestamp: "2026-09-17T18:13:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:40Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9.18d62e4f5f2e72fc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18950"
    uid: f3a29de0-3430-4d35-a668-83602fdb5072
  reason: BackOff
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-secondary-c81569-predictor-9b69cb5d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18802"
    uid: 914fc320-3c27-4e65-9f1a-344dcdd4b075
  kind: Event
  lastTimestamp: "2026-09-17T18:13:25Z"
  message: 'Created pod: isvc-secondary-c81569-predictor-9b69cb5d-q5cn9'
  metadata:
    creationTimestamp: "2026-09-17T18:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:25Z"
    name: isvc-secondary-c81569-predictor-9b69cb5d.18d62e4c04918f64
    namespace: kserve-ci-e2e-test
    resourceVersion: "18812"
    uid: 541b47fe-c528-492c-b3fd-02c1dd867224
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:25Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-secondary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18800"
    uid: f643f940-0309-4505-89f6-50517d67cf2e
  kind: Event
  lastTimestamp: "2026-09-17T18:13:25Z"
  message: Scaled up replica set isvc-secondary-c81569-predictor-9b69cb5d from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T18:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:25Z"
    name: isvc-secondary-c81569-predictor.18d62e4c01a8ebd0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18804"
    uid: c33382fe-b849-4127-9ad3-1b5a0f1cf97b
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18801"
    uid: ea31f6b5-6b61-4193-bebd-644cd5b1fe5a
  kind: Event
  lastTimestamp: "2026-09-17T18:13:40Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:13:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:40Z"
    name: isvc-secondary-c81569-predictor.18d62e4f803c74ea
    namespace: kserve-ci-e2e-test
    resourceVersion: "18961"
    uid: 0ef76ad2-25f9-43f8-8047-888f1a5747ba
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:40Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-secondary-c81569-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18801"
    uid: ea31f6b5-6b61-4193-bebd-644cd5b1fe5a
  kind: Event
  lastTimestamp: "2026-09-17T18:13:40Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:13:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:40Z"
    name: isvc-secondary-c81569-predictor.18d62e4f80a5775d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18963"
    uid: c4bce689-41ce-4d6b-847a-a796b3bbea4c
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "18793"
    uid: 6af9c758-d5c6-4b86-a138-e4ee207eb7c4
  kind: Event
  lastTimestamp: "2026-09-17T18:13:40Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:40Z"
    name: isvc-secondary-c81569.18d62e4c01a7c5ca
    namespace: kserve-ci-e2e-test
    resourceVersion: "18954"
    uid: fcc44b82-5245-4125-98bd-ff2a57e39604
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "18793"
    uid: 6af9c758-d5c6-4b86-a138-e4ee207eb7c4
  kind: Event
  lastTimestamp: "2026-09-17T18:13:25Z"
  message: 'Failed to update status for InferenceService "isvc-secondary-c81569":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c81569":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:25Z"
    name: isvc-secondary-c81569.18d62e4c02351bdc
    namespace: kserve-ci-e2e-test
    resourceVersion: "18806"
    uid: 95a8dda1-f98d-454a-b068-d2ac5e9f8c16
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:25Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-secondary-c81569
    namespace: kserve-ci-e2e-test
    resourceVersion: "18793"
    uid: 6af9c758-d5c6-4b86-a138-e4ee207eb7c4
  kind: Event
  lastTimestamp: "2026-09-17T18:13:25Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "isvc-secondary-c81569": the object has
    been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:13:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:25Z"
    name: isvc-secondary-c81569.18d62e4c023568bd
    namespace: kserve-ci-e2e-test
    resourceVersion: "18807"
    uid: 23e81cf3-a433-45e0-bf67-0923cbf34bdf
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:08:20.357983Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15576"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e04dd8b4c17
    namespace: kserve-ci-e2e-test
    resourceVersion: "15582"
    uid: 08b9318f-5d96-422a-b522-69aa3cebc206
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-0cba7-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e04ea50c793
    namespace: kserve-ci-e2e-test
    resourceVersion: "15620"
    uid: 218bd7e6-8ade-4fa9-b90e-5a10aa6b2bc9
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15587"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: Add eth0 [10.134.0.28/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e05234895e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "15638"
    uid: f1a98714-9b4e-4330-9f2e-01003dd14120
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e0524e86cb0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15640"
    uid: 1553ef23-78f0-4fae-8bfd-b3f470921353
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e0527c46e35
    namespace: kserve-ci-e2e-test
    resourceVersion: "15642"
    uid: e4f31ade-06cf-4c4f-abd6-48f48a1c41d3
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e05284bc3c6
    namespace: kserve-ci-e2e-test
    resourceVersion: "15643"
    uid: 69f418e4-28b7-4249-8bfe-834eaeabc7e6
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:25Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:25Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e062bf81f86
    namespace: kserve-ci-e2e-test
    resourceVersion: "15696"
    uid: a4b89318-f6d5-442f-b7d9-3e751d0451ed
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:26Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:08:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:26Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e062f2e00a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15700"
    uid: 75a6a960-a44b-4b97-b121-e4a9dd5c5e0c
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:26Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:08:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:26Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e062f96be0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15703"
    uid: 1521f8d5-c28e-4463-aeb1-7e173737cf53
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:26Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:26Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e062fa10b71
    namespace: kserve-ci-e2e-test
    resourceVersion: "15704"
    uid: c65ecbb9-b89b-41d8-82ff-0f61187f943e
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:26Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:08:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:26Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e06326fd3cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "15707"
    uid: 3df1b571-763a-4b04-910f-ca1c9594d14a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:26Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:26Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:08:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:26Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e0632d2b493
    namespace: kserve-ci-e2e-test
    resourceVersion: "15708"
    uid: ca2a21af-d72b-406c-8d17-52c5ecfe011c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:27Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:10:13Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:08:27Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:13Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e06a4408af8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16900"
    uid: d3695976-79ad-4dc6-8146-e06c82b3b7ae
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e1e8822e893
    namespace: kserve-ci-e2e-test
    resourceVersion: "16770"
    uid: 17a6c2d5-76b6-4d25-8047-0827fe137850
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e1e88235b2c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16776"
    uid: f5612123-ba88-4ea4-9a08-0ddbd882d5fe
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn
    namespace: kserve-ci-e2e-test
    resourceVersion: "15578"
    uid: 2bd93e17-0115-4979-883a-aef312aa68b3
  kind: Event
  lastTimestamp: "2026-09-17T18:10:13Z"
  message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp
    10.134.0.28:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:10:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:13Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn.18d62e1f529d01cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16899"
    uid: 5e76af30-dec7-45f9-b4d8-3232eff7e81c
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4
    namespace: kserve-ci-e2e-test
    resourceVersion: "15566"
    uid: a24d8ce5-83b8-4904-9696-d9a82162d846
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: 'Created pod: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4-znrvn'
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4.18d62e04dd1b8d1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "15577"
    uid: 11494640-dfbe-45f6-a775-518e2645408f
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15563"
    uid: eda07b71-fd4d-46d5-a648-1c2e2b910f24
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-0cba7-predictor-8cbb5dcb4
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor.18d62e04dac19e92
    namespace: kserve-ci-e2e-test
    resourceVersion: "15569"
    uid: ed0aff04-883b-407a-9a03-7141d7e33fdd
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:35Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15565"
    uid: 405569f0-7d97-46da-a170-2b3e79b3178e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:05Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:08:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:05Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor.18d62e0859d50641
    namespace: kserve-ci-e2e-test
    resourceVersion: "16023"
    uid: 259adde6-9d25-46d6-b0b1-a65d159a34c1
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:35Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15565"
    uid: 405569f0-7d97-46da-a170-2b3e79b3178e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:05Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:08:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:05Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor.18d62e085ab606d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16024"
    uid: aa5b0eb3-69cb-4f18-bfb6-ba03f18f2c59
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:50Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15787"
    uid: 405569f0-7d97-46da-a170-2b3e79b3178e
  kind: Event
  lastTimestamp: "2026-09-17T18:10:05Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:08:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:05Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor.18d62e0bd973f9ad
    namespace: kserve-ci-e2e-test
    resourceVersion: "16682"
    uid: 980c181e-af90-428a-88ae-22a8e5da4a9f
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:50Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15787"
    uid: 405569f0-7d97-46da-a170-2b3e79b3178e
  kind: Event
  lastTimestamp: "2026-09-17T18:10:05Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:08:50Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:05Z"
    name: isvc-sklearn-graph-raw-0cba7-predictor.18d62e0bd9e2123c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16684"
    uid: d6f92c51-d942-4eb8-9004-67fe0a4bb108
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "15552"
    uid: bd166410-64e8-4439-baea-03483973a476
  kind: Event
  lastTimestamp: "2026-09-17T18:09:44Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:09:44Z"
    name: isvc-sklearn-graph-raw-0cba7.18d62e04daa22b08
    namespace: kserve-ci-e2e-test
    resourceVersion: "16325"
    uid: 8ccdf2cf-1b03-43fc-9bd6-b377b5bf31c4
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "15552"
    uid: bd166410-64e8-4439-baea-03483973a476
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-0cba7":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-0cba7":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-sklearn-graph-raw-0cba7.18d62e04db0332dd
    namespace: kserve-ci-e2e-test
    resourceVersion: "15572"
    uid: 51f7f6fc-2a41-4b63-bc73-5f9e0bf0a09c
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:44Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16323"
    uid: bd166410-64e8-4439-baea-03483973a476
  kind: Event
  lastTimestamp: "2026-09-17T18:09:44Z"
  message: InferenceService [isvc-sklearn-graph-raw-0cba7] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:09:44Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:09:44Z"
    name: isvc-sklearn-graph-raw-0cba7.18d62e185cc97286
    namespace: kserve-ci-e2e-test
    resourceVersion: "16324"
    uid: 38855e2f-f904-4626-8f31-adb2e09590ab
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:10:10.757360Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16807"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1e91dc0f9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16813"
    uid: 3561a290-e992-450e-a432-e653c70e2094
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16825"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:11Z"
  message: Add eth0 [10.133.0.36/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:10:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:10:11Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1eb4fa3953
    namespace: kserve-ci-e2e-test
    resourceVersion: "16860"
    uid: e976065b-64e8-4bf2-997e-28d3309e063a
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:11Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:10:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:11Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1eb6be173e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16862"
    uid: dab3263c-cb5c-460a-a78b-1fbcd3ebbae8
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:11Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:10:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:11Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1eb9486242
    namespace: kserve-ci-e2e-test
    resourceVersion: "16863"
    uid: 93a41a3c-3154-4a6c-8df4-44bda4f12928
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:11Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:10:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:11Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1eb9af06d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "16864"
    uid: 23c387f0-b314-4360-9823-96fad8397885
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:15Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:15Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1faa79b117
    namespace: kserve-ci-e2e-test
    resourceVersion: "16916"
    uid: feb9009f-65be-458d-a20a-f238ec9a7fad
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:15Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:15Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1fad807cc8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16920"
    uid: 9add959d-6c9f-4547-8fa7-f895ea31a93f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:15Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:15Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1faddff884
    namespace: kserve-ci-e2e-test
    resourceVersion: "16922"
    uid: c1b2fb2e-79f5-42f2-a499-1aa0649a2b23
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:15Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:15Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1fade94d16
    namespace: kserve-ci-e2e-test
    resourceVersion: "16926"
    uid: 14de8ef0-f665-486d-b6ab-fb758f2bcf47
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:15Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:15Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1fb0f76b0b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16929"
    uid: d8d85769-ce40-4dc9-9b95-8cc503d04547
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:10:15Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:15Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1fb15dceb8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16930"
    uid: b967909c-abd3-4705-8b07-15e7ab7345c8
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:12:12Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:10:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:12Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e1fe69c2690
    namespace: kserve-ci-e2e-test
    resourceVersion: "18135"
    uid: 4404c73c-46ec-46b4-9660-e1275b532792
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:12:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:10Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e3a8f63fac1
    namespace: kserve-ci-e2e-test
    resourceVersion: "18066"
    uid: f009da15-3d7f-4ff5-8508-45112364c233
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e3a8f6495d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18077"
    uid: 34a9667a-b3bb-459f-aeca-19804d076b0a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16810"
    uid: ebb1a5a2-6151-4bdc-9568-002c61edb068
  kind: Event
  lastTimestamp: "2026-09-17T18:12:12Z"
  message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp
    10.133.0.36:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:12:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:12Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd.18d62e3ae90142cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "18134"
    uid: fab063b7-798c-4941-b6b7-17acd5720278
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664
    namespace: kserve-ci-e2e-test
    resourceVersion: "16796"
    uid: 7357706e-04d4-466a-93a0-6c0914804da5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: 'Created pod: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664-gktvd'
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664.18d62e1e90f88fbc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16809"
    uid: d3673d8d-bba7-49cd-b0bc-b8b75b36ea20
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16794"
    uid: feebcc45-756c-4019-a106-408963a7e26e
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Scaled up replica set isvc-sklearn-graph-raw-hpa-7da3c-predictor-6d49d49664
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor.18d62e1e8d46a93f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16799"
    uid: 0c1a76e7-a575-4188-9d11-0c48ec7337f9
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:25Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16795"
    uid: f2127ce5-778b-4786-acb6-0a9ddb9c82e2
  kind: Event
  lastTimestamp: "2026-09-17T18:10:55Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:10:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:55Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor.18d62e220bfc2ffe
    namespace: kserve-ci-e2e-test
    resourceVersion: "17254"
    uid: dd6c7cb0-16ec-4b65-8476-1f8fe33e9953
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:25Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16795"
    uid: f2127ce5-778b-4786-acb6-0a9ddb9c82e2
  kind: Event
  lastTimestamp: "2026-09-17T18:10:55Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:10:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:55Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor.18d62e220cb6b550
    namespace: kserve-ci-e2e-test
    resourceVersion: "17255"
    uid: 94d230ed-99a3-44f0-ab0d-f94998c4e9ae
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:10Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17011"
    uid: f2127ce5-778b-4786-acb6-0a9ddb9c82e2
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:11:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:10Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor.18d62e2c8abdc86d
    namespace: kserve-ci-e2e-test
    resourceVersion: "18041"
    uid: aec324c3-2b4f-47a2-b06e-6dcb1d7c04f9
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:10Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17011"
    uid: f2127ce5-778b-4786-acb6-0a9ddb9c82e2
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:11:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:10Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c-predictor.18d62e2c8b76b4a7
    namespace: kserve-ci-e2e-test
    resourceVersion: "18042"
    uid: 24a85907-f457-4c98-ac7d-a89aa3255ddd
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-7da3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16777"
    uid: 3b7377c9-904b-4178-b6cf-708509a768a5
  kind: Event
  lastTimestamp: "2026-09-17T18:11:42Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:42Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c.18d62e1e8d637e96
    namespace: kserve-ci-e2e-test
    resourceVersion: "17749"
    uid: 1ccb5667-f0b9-4e79-83e4-32ea27862eda
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:42Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-graph-raw-hpa-7da3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17747"
    uid: 3b7377c9-904b-4178-b6cf-708509a768a5
  kind: Event
  lastTimestamp: "2026-09-17T18:11:42Z"
  message: InferenceService [isvc-sklearn-graph-raw-hpa-7da3c] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:11:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:42Z"
    name: isvc-sklearn-graph-raw-hpa-7da3c.18d62e33f3682762
    namespace: kserve-ci-e2e-test
    resourceVersion: "17748"
    uid: 25e3ecb3-fb44-406f-ab2a-484879522e93
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:14:06.551521Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19399"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e5578497dd6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19405"
    uid: 76450984-de99-4428-9fc6-5e4ea86bae08
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:07Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19409"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:07Z"
  message: Add eth0 [10.134.0.35/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:14:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:14:07Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e559a2ed3c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19419"
    uid: f93a4c72-5ec2-4092-af60-f01e674c2bbb
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:07Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:14:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:07Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e559baeb0fd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19421"
    uid: dff33d98-3d61-4338-b004-4bb38549cc8d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:07Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:14:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:07Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e559e54e4d0
    namespace: kserve-ci-e2e-test
    resourceVersion: "19423"
    uid: 587b5419-8712-46f4-801a-f66e118308fa
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:07Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:07Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:14:07Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:07Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e559ecd46d6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19424"
    uid: 06d4324c-3017-4149-82c3-1c23fea1011c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:11Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:14:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:11Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56a020a5bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19464"
    uid: a403c2f1-ab95-4dd9-a76b-34196e79d029
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:11Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:14:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:11Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56a2fa2a24
    namespace: kserve-ci-e2e-test
    resourceVersion: "19468"
    uid: a8990841-b168-4b5b-b4e4-c16f218ba444
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:11Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:14:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:11Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56a370b9ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "19471"
    uid: bb3b9cfa-a824-43b4-832f-6ceb4ede0c08
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:11Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:14:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:11Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56a379aa74
    namespace: kserve-ci-e2e-test
    resourceVersion: "19472"
    uid: 11b38c58-57e4-43fc-b619-bd4a14c1a03f
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:11Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:14:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:11Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56a68edac0
    namespace: kserve-ci-e2e-test
    resourceVersion: "19475"
    uid: 0d08a4e4-a04f-49aa-84d6-c1e09d8240bc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:14:11Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:14:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:11Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56a703f38f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19476"
    uid: 5f3e04c1-400d-44e3-9776-4ac06bb4ec51
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 20
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:16:58Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:14:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:16:58Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e56dc28c07e
    namespace: kserve-ci-e2e-test
    resourceVersion: "20821"
    uid: 490cc815-2492-4d42-b0b4-3c0821292ee5
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:16:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:16:56Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:16:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:16:56Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e7d1144b5f7
    namespace: kserve-ci-e2e-test
    resourceVersion: "20807"
    uid: abaa9905-9cec-435b-87d3-c88dc9f9c7f8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:16:56Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:16:56Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:16:56Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:16:56Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e7d1145c9f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20808"
    uid: ba0408bb-e805-4bc3-987e-dac7d0a7f214
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:16:59Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19400"
    uid: 3ea0501d-7e40-4ff8-946d-cf5bc861bf9c
  kind: Event
  lastTimestamp: "2026-09-17T18:17:04Z"
  message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp
    10.134.0.35:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:16:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:04Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4.18d62e7dbe99b3ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "20974"
    uid: 254f09ef-28e5-4fe5-9f08-76fb68b75bb7
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "19390"
    uid: 0a8f64d8-e229-40a6-8fbe-97bf100cf1e4
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: 'Created pod: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd-kqzp4'
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd.18d62e5577d80be7
    namespace: kserve-ci-e2e-test
    resourceVersion: "19401"
    uid: 94a90f84-2a14-4ca5-993a-ea113a6c0f79
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-sklearn-scale-raw-91a22-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19387"
    uid: 9204793d-8a3c-4185-a290-40cfc4745cb2
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: Scaled up replica set isvc-sklearn-scale-raw-91a22-predictor-85679b8fcd
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: isvc-sklearn-scale-raw-91a22-predictor.18d62e55752798a1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19392"
    uid: 8772658e-a1a3-4ba9-accc-6688b84c9c06
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-91a22-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19388"
    uid: 01ebe2c9-44c1-423c-90c8-46ff3b38d355
  kind: Event
  lastTimestamp: "2026-09-17T18:14:51Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:14:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:51Z"
    name: isvc-sklearn-scale-raw-91a22-predictor.18d62e58f375eade
    namespace: kserve-ci-e2e-test
    resourceVersion: "19779"
    uid: 7c867867-7c83-4d5d-9e87-26dd575c0620
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:21Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-91a22-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19388"
    uid: 01ebe2c9-44c1-423c-90c8-46ff3b38d355
  kind: Event
  lastTimestamp: "2026-09-17T18:14:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:14:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:51Z"
    name: isvc-sklearn-scale-raw-91a22-predictor.18d62e58f3d79102
    namespace: kserve-ci-e2e-test
    resourceVersion: "19780"
    uid: 91f5900a-37a9-4a9e-9411-1bf45f3aa8a7
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-91a22-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19557"
    uid: 01ebe2c9-44c1-423c-90c8-46ff3b38d355
  kind: Event
  lastTimestamp: "2026-09-17T18:16:51Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:15:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:16:51Z"
    name: isvc-sklearn-scale-raw-91a22-predictor.18d62e6370083590
    namespace: kserve-ci-e2e-test
    resourceVersion: "20757"
    uid: c0a4853b-e4d4-43d4-8397-f48e2ee3970e
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-sklearn-scale-raw-91a22-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19557"
    uid: 01ebe2c9-44c1-423c-90c8-46ff3b38d355
  kind: Event
  lastTimestamp: "2026-09-17T18:16:51Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:15:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:16:51Z"
    name: isvc-sklearn-scale-raw-91a22-predictor.18d62e63706bb6a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20758"
    uid: 56ddfd02-af9a-4b34-bc31-28f6dad080e8
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-91a22
    namespace: kserve-ci-e2e-test
    resourceVersion: "19372"
    uid: 82ccfeed-d5d4-438e-8a9b-107650350dae
  kind: Event
  lastTimestamp: "2026-09-17T18:16:48Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:16:48Z"
    name: isvc-sklearn-scale-raw-91a22.18d62e55752625da
    namespace: kserve-ci-e2e-test
    resourceVersion: "20732"
    uid: b168b9ac-040e-4693-94dd-efe755a322ac
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:16:48Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-sklearn-scale-raw-91a22
    namespace: kserve-ci-e2e-test
    resourceVersion: "20729"
    uid: 82ccfeed-d5d4-438e-8a9b-107650350dae
  kind: Event
  lastTimestamp: "2026-09-17T18:16:48Z"
  message: InferenceService [isvc-sklearn-scale-raw-91a22] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:16:48Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:16:48Z"
    name: isvc-sklearn-scale-raw-91a22.18d62e7b168fd2d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20730"
    uid: d4a6e60e-7775-4b0f-a737-989a6e7d882c
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:08:20.507904Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15606"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e04e67af83a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15611"
    uid: 9e6d7555-7885-40c0-bfc3-eaac0f3aab42
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-0cba7-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e04f054fdcf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15624"
    uid: 17f946f9-da90-4dbe-a1f5-60da3dfc84fd
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15617"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: Add eth0 [10.134.0.29/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e052cc29ff0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15645"
    uid: a0aa8599-f245-45ec-a99d-b2c86d61adb1
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e052e5283de
    namespace: kserve-ci-e2e-test
    resourceVersion: "15647"
    uid: 964c2831-50df-44d8-a02b-178b0c66f9b9
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0531de0cd3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15648"
    uid: edd3bb0b-2e3f-4165-a1ce-93064d04afa2
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:21Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:21Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:08:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:21Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e053249adb0
    namespace: kserve-ci-e2e-test
    resourceVersion: "15649"
    uid: 11a2433d-e527-45b7-a254-2a9f0c48f565
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:25Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:25Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-09-17T18:08:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:25Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e062c143153
    namespace: kserve-ci-e2e-test
    resourceVersion: "15697"
    uid: 305a2293-d5fc-4330-8a81-9ba2d51e13f8
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:47Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.344s (21.344s
    including waiting). Image size: 1435742979 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:08:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:47Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b244bce9a
    namespace: kserve-ci-e2e-test
    resourceVersion: "15873"
    uid: aed1f1f0-ae47-4b40-b37f-27ee45fbaf3d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:47Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:08:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:47Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b2863cb6c
    namespace: kserve-ci-e2e-test
    resourceVersion: "15875"
    uid: f539de96-f458-4d4a-937d-e5531fd47e19
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:47Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:08:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:47Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b28ce2eb9
    namespace: kserve-ci-e2e-test
    resourceVersion: "15876"
    uid: 7048895e-a443-4025-8aef-322f52e46352
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:47Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:08:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:47Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b28db72cf
    namespace: kserve-ci-e2e-test
    resourceVersion: "15877"
    uid: 3fa58220-59f7-4327-9818-c038bf4577b7
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:47Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:08:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:47Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b2bf0e3a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "15878"
    uid: 5b4f6427-595a-4603-9e1a-ce0fb32dded5
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:47Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:08:47Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:08:47Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:08:47Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b2c5b1d6d
    namespace: kserve-ci-e2e-test
    resourceVersion: "15879"
    uid: 9aa282ee-5f11-4721-96de-0653edfc694b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 8
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:49Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:45Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:08:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:09:45Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e0b8f1606c2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16334"
    uid: 7b761e27-a365-4c45-b8cd-064488c4f24e
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e1e8a8bbc58
    namespace: kserve-ci-e2e-test
    resourceVersion: "16787"
    uid: d374cc42-d4bb-4157-9170-ff18d07e88a8
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n
    namespace: kserve-ci-e2e-test
    resourceVersion: "15608"
    uid: 0fa9a63c-679c-4161-b889-674a6a4c198e
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n.18d62e1e8a8c5335
    namespace: kserve-ci-e2e-test
    resourceVersion: "16788"
    uid: b9c3388f-4ec2-467b-acd4-e6bac2dcc94d
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685
    namespace: kserve-ci-e2e-test
    resourceVersion: "15598"
    uid: d4499fa5-5405-4924-84bc-83b3df02f4d3
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: 'Created pod: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685-slt9n'
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor-569fd45685.18d62e04e60a8362
    namespace: kserve-ci-e2e-test
    resourceVersion: "15609"
    uid: 98212f3e-3c5e-4d60-b5ba-f061df0ed55e
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15595"
    uid: 6a438f3d-9ce7-4316-b3d5-fe377a8a509b
  kind: Event
  lastTimestamp: "2026-09-17T18:08:20Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-0cba7-predictor-569fd45685
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:08:20Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor.18d62e04e339a554
    namespace: kserve-ci-e2e-test
    resourceVersion: "15600"
    uid: 9a715d98-1479-4c43-a647-ab27c89b925f
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:35Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15597"
    uid: 60922ae6-b606-4dbf-88e9-169b6d4afe54
  kind: Event
  lastTimestamp: "2026-09-17T18:09:20Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:08:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:20Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor.18d62e0861ca6135
    namespace: kserve-ci-e2e-test
    resourceVersion: "16136"
    uid: ba27027f-a25f-4775-bf7f-22ff9a7a9086
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:35Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15597"
    uid: 60922ae6-b606-4dbf-88e9-169b6d4afe54
  kind: Event
  lastTimestamp: "2026-09-17T18:09:20Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:08:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:09:20Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor.18d62e086252b523
    namespace: kserve-ci-e2e-test
    resourceVersion: "16138"
    uid: 24718dde-3f5e-42b3-85fc-0a7b7fb78985
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:35Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15790"
    uid: 60922ae6-b606-4dbf-88e9-169b6d4afe54
  kind: Event
  lastTimestamp: "2026-09-17T18:10:05Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:09:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:05Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor.18d62e165e610e4b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16688"
    uid: 0657167e-6848-4d64-a481-c14791b20194
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:35Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-0cba7-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "15790"
    uid: 60922ae6-b606-4dbf-88e9-169b6d4afe54
  kind: Event
  lastTimestamp: "2026-09-17T18:10:05Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:09:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:05Z"
    name: isvc-xgboost-graph-raw-0cba7-predictor.18d62e165f133e14
    namespace: kserve-ci-e2e-test
    resourceVersion: "16689"
    uid: 4097f81f-3895-4028-b05a-06e65c2236f3
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:08:20Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "15588"
    uid: f3b96e15-83dd-4db9-a455-91f6d79b732e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:55Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:08:20Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:09:55Z"
    name: isvc-xgboost-graph-raw-0cba7.18d62e04e304b65a
    namespace: kserve-ci-e2e-test
    resourceVersion: "16538"
    uid: 94973ffa-a0d8-4738-abea-2ff3c4d8031d
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:09:55Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16536"
    uid: f3b96e15-83dd-4db9-a455-91f6d79b732e
  kind: Event
  lastTimestamp: "2026-09-17T18:09:55Z"
  message: InferenceService [isvc-xgboost-graph-raw-0cba7] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:09:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:09:55Z"
    name: isvc-xgboost-graph-raw-0cba7.18d62e1af2f81b41
    namespace: kserve-ci-e2e-test
    resourceVersion: "16537"
    uid: 974ec6e5-906c-48a2-a975-3d77d94f7667
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:10:10.855673Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16834"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1e97b83642
    namespace: kserve-ci-e2e-test
    resourceVersion: "16839"
    uid: 30330fd3-834c-471a-86a9-b8ba92dc4117
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:11Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-7da3c-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:10:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:11Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1ea44f2a21
    namespace: kserve-ci-e2e-test
    resourceVersion: "16854"
    uid: 2ef01086-4bcd-4bf7-b814-bbf110b556c1
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16844"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:12Z"
  message: Add eth0 [10.133.0.37/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:10:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:10:12Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1eddc53c25
    namespace: kserve-ci-e2e-test
    resourceVersion: "16877"
    uid: 69d7590e-d173-4b6d-9921-6c035d3757a7
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:12Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:10:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:12Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1edf791a33
    namespace: kserve-ci-e2e-test
    resourceVersion: "16879"
    uid: 96526eb6-7d37-4b6a-91b2-9787ad26236b
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:12Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:10:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:12Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1ee36aaad6
    namespace: kserve-ci-e2e-test
    resourceVersion: "16880"
    uid: 077515eb-8cf8-4e94-b187-c0f31c1e9d6b
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:12Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:10:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:12Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1ee3cd36d2
    namespace: kserve-ci-e2e-test
    resourceVersion: "16881"
    uid: a37cd4c5-17e2-4895-8b33-5d0660de9f3c
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:16Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:16Z"
  message: Pulling image "kserve/xgbserver:latest"
  metadata:
    creationTimestamp: "2026-09-17T18:10:16Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:16Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e1fe677c329
    namespace: kserve-ci-e2e-test
    resourceVersion: "16937"
    uid: 03c40f76-f4a3-4e28-8a22-28cea67c5bd9
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:38Z"
  message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.889s (21.889s
    including waiting). Image size: 1435742979 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:10:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:38Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e24ff2fd4ec
    namespace: kserve-ci-e2e-test
    resourceVersion: "17112"
    uid: 17ca96dd-5090-477e-ab1c-264ecd298ff1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:38Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:10:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:38Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e25028d9ecb
    namespace: kserve-ci-e2e-test
    resourceVersion: "17114"
    uid: 48f0ef5f-a8eb-4cb2-8a2c-d42dd404a813
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:38Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:10:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:38Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e2502fb65f3
    namespace: kserve-ci-e2e-test
    resourceVersion: "17115"
    uid: 76a9e0f8-5fa3-4797-a1e0-815c8a45949e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:38Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:10:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:38Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e2503052561
    namespace: kserve-ci-e2e-test
    resourceVersion: "17116"
    uid: 7d08aea3-6610-415d-919c-f2008c723e85
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:38Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:10:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:38Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e2505fa5f98
    namespace: kserve-ci-e2e-test
    resourceVersion: "17117"
    uid: bf4882b0-d076-4531-b9f2-e1dc88ae0a86
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:38Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:10:38Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:10:38Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:38Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e25065c3341
    namespace: kserve-ci-e2e-test
    resourceVersion: "17118"
    uid: 54428b03-5350-4a26-b265-b24172fee95d
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 9
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:39Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:11:45Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:10:39Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:11:45Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e25493196eb
    namespace: kserve-ci-e2e-test
    resourceVersion: "17781"
    uid: 33b023dc-2fb5-4c43-b793-2accd9800a90
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:12:11Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e3a950e620c
    namespace: kserve-ci-e2e-test
    resourceVersion: "18083"
    uid: 563173c6-722a-4916-afa1-3217bd0aee13
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:11Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt
    namespace: kserve-ci-e2e-test
    resourceVersion: "16836"
    uid: 5f2fc54b-3b28-4f25-a085-766376c72102
  kind: Event
  lastTimestamp: "2026-09-17T18:12:11Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt.18d62e3a950ef330
    namespace: kserve-ci-e2e-test
    resourceVersion: "18087"
    uid: c012a764-0f5e-45cb-aeca-6ccc4b8179c3
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f
    namespace: kserve-ci-e2e-test
    resourceVersion: "16824"
    uid: ab61d14c-33ca-4bd8-be51-29418d5ad9cb
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: 'Created pod: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f-jppwt'
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f.18d62e1e9720410e
    namespace: kserve-ci-e2e-test
    resourceVersion: "16835"
    uid: ca82a9b5-12b7-457a-9982-b2613d604668
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16821"
    uid: f0158e8d-f84e-4021-af5a-e3e5b065e641
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Scaled up replica set isvc-xgboost-graph-raw-hpa-7da3c-predictor-ffb8dcf4f
    from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor.18d62e1e94621134
    namespace: kserve-ci-e2e-test
    resourceVersion: "16826"
    uid: 4d5f738a-07c5-4bf2-b2fd-f5ad61bb7737
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:25Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16823"
    uid: b93fa61f-90e2-43f9-aeb2-95ea932996f1
  kind: Event
  lastTimestamp: "2026-09-17T18:11:25Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:10:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:11:25Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor.18d62e2212ef90b2
    namespace: kserve-ci-e2e-test
    resourceVersion: "17539"
    uid: 532c0b62-d35d-412d-9150-a9463a96df6a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:25Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "16823"
    uid: b93fa61f-90e2-43f9-aeb2-95ea932996f1
  kind: Event
  lastTimestamp: "2026-09-17T18:11:25Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:10:25Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:11:25Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor.18d62e22135cccb4
    namespace: kserve-ci-e2e-test
    resourceVersion: "17540"
    uid: 257f8d72-9d58-409b-a7ec-5723b5b5d5de
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:10Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17014"
    uid: b93fa61f-90e2-43f9-aeb2-95ea932996f1
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:11:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:10Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor.18d62e2c8fda61a3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18062"
    uid: d20df447-1ecf-42ea-b3ec-8004b133dd08
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 4
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:10Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "17014"
    uid: b93fa61f-90e2-43f9-aeb2-95ea932996f1
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:11:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:10Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c-predictor.18d62e2c906adbee
    namespace: kserve-ci-e2e-test
    resourceVersion: "18067"
    uid: 632bc701-46b0-4067-a9e2-ad8cfc1f69b6
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-7da3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "16806"
    uid: dafaa61c-f4cd-45f6-8f4b-9fba0e820092
  kind: Event
  lastTimestamp: "2026-09-17T18:11:55Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:55Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c.18d62e1e948eccc1
    namespace: kserve-ci-e2e-test
    resourceVersion: "17875"
    uid: 714a9b4e-01f2-444a-af8c-9576bc3625af
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:11:55Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: isvc-xgboost-graph-raw-hpa-7da3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17873"
    uid: dafaa61c-f4cd-45f6-8f4b-9fba0e820092
  kind: Event
  lastTimestamp: "2026-09-17T18:11:55Z"
  message: InferenceService [isvc-xgboost-graph-raw-hpa-7da3c] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:11:55Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:11:55Z"
    name: isvc-xgboost-graph-raw-hpa-7da3c.18d62e36ffcf8379
    namespace: kserve-ci-e2e-test
    resourceVersion: "17874"
    uid: 837588dc-bc96-45f7-b521-347779ae7307
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:21:51.819628Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23392"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:21:51Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:21:51Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1cc6d671a
    namespace: kserve-ci-e2e-test
    resourceVersion: "23396"
    uid: 5f1020cd-3420-46af-9615-62849679a931
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23400"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: Add eth0 [10.133.0.44/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1ee5784fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "23410"
    uid: 60ebbd4a-08b0-4d13-b85a-ff92ae7dc5d0
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: Container image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1efb7c862
    namespace: kserve-ci-e2e-test
    resourceVersion: "23412"
    uid: 35ed2281-c655-4fdf-b006-ea2649818026
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1f3f6ee7b
    namespace: kserve-ci-e2e-test
    resourceVersion: "23414"
    uid: b5533f1d-6194-47bb-9d8b-43994284db22
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1f4660a08
    namespace: kserve-ci-e2e-test
    resourceVersion: "23415"
    uid: 4b628054-d20b-4a6e-bbdb-24366f1af260
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1f470f241
    namespace: kserve-ci-e2e-test
    resourceVersion: "23416"
    uid: f19e83cd-c47a-44f8-a22b-5b2bfba82d17
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1f6cab293
    namespace: kserve-ci-e2e-test
    resourceVersion: "23417"
    uid: f1c51ddc-29b3-43a8-8f22-4ab02f269cd8
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:52Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:21:52Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:21:52Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:21:52Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62ec1f73a0199
    namespace: kserve-ci-e2e-test
    resourceVersion: "23418"
    uid: ae3cf35c-1018-4f86-a704-e9f4e91e6b10
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:23:37Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:23:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:37Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62eda4ee8b3a4
    namespace: kserve-ci-e2e-test
    resourceVersion: "24278"
    uid: d06a8dd5-c0a4-4287-9892-988fb01e8236
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:23:37Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv
    namespace: kserve-ci-e2e-test
    resourceVersion: "23394"
    uid: c60002b5-3e77-4688-bbc1-3367779d5fcc
  kind: Event
  lastTimestamp: "2026-09-17T18:23:37Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:23:37Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:23:37Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv.18d62eda4ee9e232
    namespace: kserve-ci-e2e-test
    resourceVersion: "24279"
    uid: fbc784f7-82a8-4437-a608-4a52d8e77fee
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-cipn-predictor-9774bf55d
    namespace: kserve-ci-e2e-test
    resourceVersion: "23381"
    uid: a556ae03-50b6-4006-b903-e502811b0507
  kind: Event
  lastTimestamp: "2026-09-17T18:21:51Z"
  message: 'Created pod: message-dumper-raw-cipn-predictor-9774bf55d-lwcfv'
  metadata:
    creationTimestamp: "2026-09-17T18:21:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:21:51Z"
    name: message-dumper-raw-cipn-predictor-9774bf55d.18d62ec1cc006046
    namespace: kserve-ci-e2e-test
    resourceVersion: "23393"
    uid: b417876b-e770-4762-904e-eff548f88491
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:51Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23379"
    uid: 52195782-9f61-459b-92da-af9f484e4702
  kind: Event
  lastTimestamp: "2026-09-17T18:21:51Z"
  message: Scaled up replica set message-dumper-raw-cipn-predictor-9774bf55d from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:21:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:21:51Z"
    name: message-dumper-raw-cipn-predictor.18d62ec1c8cdb200
    namespace: kserve-ci-e2e-test
    resourceVersion: "23384"
    uid: ee11bbca-e8a9-471f-bdb7-b2f1ad9b34b3
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23380"
    uid: 5dd5dd30-e9ff-4321-b1da-234316f385c9
  kind: Event
  lastTimestamp: "2026-09-17T18:22:36Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:22:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:22:36Z"
    name: message-dumper-raw-cipn-predictor.18d62ec547ae276f
    namespace: kserve-ci-e2e-test
    resourceVersion: "23807"
    uid: a217e654-ed4f-411d-901e-60f4b9e58717
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:06Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-cipn-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "23380"
    uid: 5dd5dd30-e9ff-4321-b1da-234316f385c9
  kind: Event
  lastTimestamp: "2026-09-17T18:22:36Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:22:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:22:36Z"
    name: message-dumper-raw-cipn-predictor.18d62ec5480cfc07
    namespace: kserve-ci-e2e-test
    resourceVersion: "23808"
    uid: 45f9280e-ace3-471a-ae9a-fbc7f9eae631
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23373"
    uid: 5c56b4d0-fbc1-4976-bd14-7a1eb986b269
  kind: Event
  lastTimestamp: "2026-09-17T18:22:01Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:21:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:22:01Z"
    name: message-dumper-raw-cipn.18d62ec1c8f80c4c
    namespace: kserve-ci-e2e-test
    resourceVersion: "23485"
    uid: d93f2b3b-c435-46f7-82f0-f2752c94eeb0
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:21:51Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23373"
    uid: 5c56b4d0-fbc1-4976-bd14-7a1eb986b269
  kind: Event
  lastTimestamp: "2026-09-17T18:21:51Z"
  message: 'Failed to update status for InferenceService "message-dumper-raw-cipn":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cipn":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:21:51Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:21:51Z"
    name: message-dumper-raw-cipn.18d62ec1c9550ebb
    namespace: kserve-ci-e2e-test
    resourceVersion: "23387"
    uid: db0016a8-a7cd-42a3-9c59-95b738e38554
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:22:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-cipn
    namespace: kserve-ci-e2e-test
    resourceVersion: "23483"
    uid: 5c56b4d0-fbc1-4976-bd14-7a1eb986b269
  kind: Event
  lastTimestamp: "2026-09-17T18:22:00Z"
  message: InferenceService [message-dumper-raw-cipn] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:22:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:22:00Z"
    name: message-dumper-raw-cipn.18d62ec3ef4215e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "23484"
    uid: 29b0c5b8-25a4-4978-b983-a5f82c1deff8
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:12:11.142471Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18100"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3a995f1b1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18105"
    uid: 40e6f2e4-fa7e-42d3-8d77-f318015d466a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:11Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:11Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-fc652-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3aa3de9e54
    namespace: kserve-ci-e2e-test
    resourceVersion: "18116"
    uid: 159fc91f-f7a7-47a7-a92e-fc3ffed98805
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:12Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18111"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:12Z"
  message: Add eth0 [10.133.0.38/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:12:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:12:12Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3adee7812f
    namespace: kserve-ci-e2e-test
    resourceVersion: "18129"
    uid: cd5dc539-d129-44b2-ae20-c08b849918bd
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:12Z"
  message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
  metadata:
    creationTimestamp: "2026-09-17T18:12:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:12Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3ae0588f03
    namespace: kserve-ci-e2e-test
    resourceVersion: "18131"
    uid: 71318936-2643-4cdb-8f83-04aef23cb3e4
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:13Z"
  message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display"
    in 1.101s (1.101s including waiting). Image size: 14813193 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:12:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:13Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3b22042e51
    namespace: kserve-ci-e2e-test
    resourceVersion: "18142"
    uid: 60e4d0db-4828-4d33-8dba-fb85ccbd0ec6
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:13Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:12:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:13Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3b252e0dec
    namespace: kserve-ci-e2e-test
    resourceVersion: "18145"
    uid: 03a371f2-b371-4401-8b3a-a865797bba75
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:13Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:12:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:13Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3b258f7c34
    namespace: kserve-ci-e2e-test
    resourceVersion: "18146"
    uid: 4b094209-68cd-464c-a3a7-a9237da5609a
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:13Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:12:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:13Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3b2599f743
    namespace: kserve-ci-e2e-test
    resourceVersion: "18147"
    uid: 84a8d9f9-9f98-4b72-a5f2-8a0b83ddc4d1
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:13Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:12:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:13Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3b286e2a3e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18148"
    uid: 7f975065-4fad-4ea9-bd79-ea8e3bcb4a61
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:13Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:12:13Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:12:13Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:13Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e3b28ce1237
    namespace: kserve-ci-e2e-test
    resourceVersion: "18149"
    uid: c4e1740d-30bd-46c3-be23-66ba95084f4b
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e5572df1de2
    namespace: kserve-ci-e2e-test
    resourceVersion: "19379"
    uid: 21a10e3b-161b-4ba2-b4e5-a34869fb0cec
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:06Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82
    namespace: kserve-ci-e2e-test
    resourceVersion: "18102"
    uid: d5332680-2e45-4515-8f92-ed7c03120327
  kind: Event
  lastTimestamp: "2026-09-17T18:14:06Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:14:06Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:06Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82.18d62e5572e02236
    namespace: kserve-ci-e2e-test
    resourceVersion: "19381"
    uid: 193d8bac-cdd3-4681-a3e6-66bd52558c49
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: message-dumper-raw-fc652-predictor-6d55ffc99b
    namespace: kserve-ci-e2e-test
    resourceVersion: "18091"
    uid: 226e731c-5183-4e4d-a0d3-9b99bb19efaf
  kind: Event
  lastTimestamp: "2026-09-17T18:12:11Z"
  message: 'Created pod: message-dumper-raw-fc652-predictor-6d55ffc99b-44n82'
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: message-dumper-raw-fc652-predictor-6d55ffc99b.18d62e3a98f34f1e
    namespace: kserve-ci-e2e-test
    resourceVersion: "18101"
    uid: 76033db2-5e8d-4ba0-82f6-2d721029327c
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:11Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: message-dumper-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18088"
    uid: a91d769b-9281-4f94-b860-016d442f938c
  kind: Event
  lastTimestamp: "2026-09-17T18:12:11Z"
  message: Scaled up replica set message-dumper-raw-fc652-predictor-6d55ffc99b from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:11Z"
    name: message-dumper-raw-fc652-predictor.18d62e3a970743b3
    namespace: kserve-ci-e2e-test
    resourceVersion: "18094"
    uid: e60f807e-0ed3-48bf-baa5-e0033aacdf5c
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18089"
    uid: fc9f39b5-6bba-4491-a965-74317139c3ee
  kind: Event
  lastTimestamp: "2026-09-17T18:12:56Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:12:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:56Z"
    name: message-dumper-raw-fc652-predictor.18d62e3e15721206
    namespace: kserve-ci-e2e-test
    resourceVersion: "18558"
    uid: d465cd0a-3601-436a-b90a-beef2d3dd3d2
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:26Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: message-dumper-raw-fc652-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "18089"
    uid: fc9f39b5-6bba-4491-a965-74317139c3ee
  kind: Event
  lastTimestamp: "2026-09-17T18:12:56Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:12:26Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:56Z"
    name: message-dumper-raw-fc652-predictor.18d62e3e15d8d755
    namespace: kserve-ci-e2e-test
    resourceVersion: "18559"
    uid: a6aab1f4-396c-4708-80e2-f0cabdecffd2
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 7
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:11Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-fc652
    namespace: kserve-ci-e2e-test
    resourceVersion: "18072"
    uid: 440adff4-7ae5-453c-b4b8-eb19466e687c
  kind: Event
  lastTimestamp: "2026-09-17T18:12:21Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:12:11Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:12:21Z"
    name: message-dumper-raw-fc652.18d62e3a9701c7c0
    namespace: kserve-ci-e2e-test
    resourceVersion: "18224"
    uid: 6c46f67b-0f53-47a8-8449-9133af54faf1
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:21Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: message-dumper-raw-fc652
    namespace: kserve-ci-e2e-test
    resourceVersion: "18222"
    uid: 440adff4-7ae5-453c-b4b8-eb19466e687c
  kind: Event
  lastTimestamp: "2026-09-17T18:12:21Z"
  message: InferenceService [message-dumper-raw-fc652] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:12:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:12:21Z"
    name: message-dumper-raw-fc652.18d62e3cf1179da2
    namespace: kserve-ci-e2e-test
    resourceVersion: "18223"
    uid: 6f7aa4a8-d12d-41fc-b41d-0da5fadaec55
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:10:00.393990Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16615"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-0cba7-854bf9fb4-zdvv8
    to ip-10-0-141-52.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:10:00Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:10:00Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1c2827ab88
    namespace: kserve-ci-e2e-test
    resourceVersion: "16620"
    uid: 4ebf147c-aa25-4f5b-b706-f831645a242a
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:00Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16624"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:00Z"
  message: Add eth0 [10.132.0.19/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:10:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:10:00Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1c4a205da7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16638"
    uid: 0c94cb9e-3266-4583-b549-72ba71c1a16c
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-0cba7}
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16616"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:00Z"
  message: Pulling image "quay.io/opendatahub/kserve-router@sha256:0c87e1a125451a8a0f1f46fadc66ac5afc05f5cb73608aee9bffb1f827822f57"
  metadata:
    creationTimestamp: "2026-09-17T18:10:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:00Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1c4b80b5e5
    namespace: kserve-ci-e2e-test
    resourceVersion: "16640"
    uid: dcfc8164-8f60-4004-963c-bc8734c5343d
  reason: Pulling
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-0cba7}
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16616"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:03Z"
  message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:0c87e1a125451a8a0f1f46fadc66ac5afc05f5cb73608aee9bffb1f827822f57"
    in 2.448s (2.448s including waiting). Image size: 211751503 bytes.'
  metadata:
    creationTimestamp: "2026-09-17T18:10:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:03Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1cdd790907
    namespace: kserve-ci-e2e-test
    resourceVersion: "16661"
    uid: a04aa99d-6c19-415b-8407-2644b4c516d5
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-0cba7}
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16616"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:03Z"
  message: 'Created container: model-chainer-raw-0cba7'
  metadata:
    creationTimestamp: "2026-09-17T18:10:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:03Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1ce070cafc
    namespace: kserve-ci-e2e-test
    resourceVersion: "16662"
    uid: 37f839fa-e324-4bc4-94d4-3bf567fb380a
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:03Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-0cba7}
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16616"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:03Z"
  message: Started container model-chainer-raw-0cba7
  metadata:
    creationTimestamp: "2026-09-17T18:10:03Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:03Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1ce0e5cecf
    namespace: kserve-ci-e2e-test
    resourceVersion: "16663"
    uid: 039b6195-a077-46fa-a874-4fca5eb7887e
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-0cba7}
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16616"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:10Z"
  message: Stopping container model-chainer-raw-0cba7
  metadata:
    creationTimestamp: "2026-09-17T18:10:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:10Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1e8183668b
    namespace: kserve-ci-e2e-test
    resourceVersion: "16744"
    uid: 60b11956-1ba2-4868-9b4a-cca8c490492a
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:15Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-0cba7}
    kind: Pod
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8
    namespace: kserve-ci-e2e-test
    resourceVersion: "16616"
    uid: 6ba46426-0f29-419e-898c-2aece26dece5
  kind: Event
  lastTimestamp: "2026-09-17T18:10:40Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T18:10:15Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:10:40Z"
    name: model-chainer-raw-0cba7-854bf9fb4-zdvv8.18d62e1f96955d5f
    namespace: kserve-ci-e2e-test
    resourceVersion: "17133"
    uid: b65f8503-ee62-4114-bc78-48d587c9ad36
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:00Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-0cba7-854bf9fb4
    namespace: kserve-ci-e2e-test
    resourceVersion: "16611"
    uid: 18d6aeaa-4372-4cf5-aab8-a6fb9c7447bf
  kind: Event
  lastTimestamp: "2026-09-17T18:10:00Z"
  message: 'Created pod: model-chainer-raw-0cba7-854bf9fb4-zdvv8'
  metadata:
    creationTimestamp: "2026-09-17T18:10:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:00Z"
    name: model-chainer-raw-0cba7-854bf9fb4.18d62e1c27ba8baf
    namespace: kserve-ci-e2e-test
    resourceVersion: "16617"
    uid: 0ed0cff5-cdd7-4c27-b482-aa858e9463c1
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:00Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16609"
    uid: 2ed83312-c72a-41ea-9a16-b5f156b4c1dc
  kind: Event
  lastTimestamp: "2026-09-17T18:10:00Z"
  message: Scaled up replica set model-chainer-raw-0cba7-854bf9fb4 from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:10:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:10:00Z"
    name: model-chainer-raw-0cba7.18d62e1c26da6949
    namespace: kserve-ci-e2e-test
    resourceVersion: "16613"
    uid: 3b316f39-09cf-4ebf-8a86-f29a1bfeb9f4
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:10:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-0cba7
    namespace: kserve-ci-e2e-test
    resourceVersion: "16626"
    uid: 84eebb94-5d88-4d55-a6b1-751760a44c76
  kind: Event
  lastTimestamp: "2026-09-17T18:10:00Z"
  message: InferenceGraph [model-chainer-raw-0cba7] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:10:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:10:00Z"
    name: model-chainer-raw-0cba7.18d62e1c2d189dbd
    namespace: kserve-ci-e2e-test
    resourceVersion: "16628"
    uid: f7e86776-4f30-4661-9826-8eb4f4787bf1
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:12:00.732382Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17926"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    to ip-10-0-141-52.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:12:00Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:12:00Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e382ce1be91
    namespace: kserve-ci-e2e-test
    resourceVersion: "17931"
    uid: 8ecd7b96-f0f8-4498-a93c-8c6f7669f7d8
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:01Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17935"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: "2026-09-17T18:12:01Z"
  message: Add eth0 [10.132.0.20/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:12:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:12:01Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e384ebb0e20
    namespace: kserve-ci-e2e-test
    resourceVersion: "17947"
    uid: c5db2367-2c81-4e25-bee0-1c2fbef91427
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-7da3c}
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17927"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: "2026-09-17T18:12:01Z"
  message: Container image "quay.io/opendatahub/kserve-router@sha256:0c87e1a125451a8a0f1f46fadc66ac5afc05f5cb73608aee9bffb1f827822f57"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:12:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:01Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e3850125b2a
    namespace: kserve-ci-e2e-test
    resourceVersion: "17949"
    uid: 1ef80f0e-c6ab-4070-99f8-f8e0afff39a6
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-7da3c}
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17927"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: "2026-09-17T18:12:01Z"
  message: 'Created container: model-chainer-raw-hpa-7da3c'
  metadata:
    creationTimestamp: "2026-09-17T18:12:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:01Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e3852c41234
    namespace: kserve-ci-e2e-test
    resourceVersion: "17951"
    uid: ec3b8838-d534-41c5-89e5-db51887290b9
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:01Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-7da3c}
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17927"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: "2026-09-17T18:12:01Z"
  message: Started container model-chainer-raw-hpa-7da3c
  metadata:
    creationTimestamp: "2026-09-17T18:12:01Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:01Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e38533aefcd
    namespace: kserve-ci-e2e-test
    resourceVersion: "17952"
    uid: d22f4ce5-08ec-47cc-bc7d-a9f3bcc73226
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:10Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-7da3c}
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17927"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: "2026-09-17T18:12:10Z"
  message: Stopping container model-chainer-raw-hpa-7da3c
  metadata:
    creationTimestamp: "2026-09-17T18:12:10Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:10Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e3a84a0a2be
    namespace: kserve-ci-e2e-test
    resourceVersion: "18039"
    uid: 5d106065-dacc-40b0-99fa-591814361263
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Normal
- apiVersion: v1
  count: 6
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:12Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{model-chainer-raw-hpa-7da3c}
    kind: Pod
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf
    namespace: kserve-ci-e2e-test
    resourceVersion: "17927"
    uid: cfc56e3b-5e4f-4fda-8325-928384e65afa
  kind: Event
  lastTimestamp: "2026-09-17T18:12:37Z"
  message: 'Readiness probe failed: HTTP probe failed with statuscode: 503'
  metadata:
    creationTimestamp: "2026-09-17T18:12:12Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:12:37Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf.18d62e3aeccaebec
    namespace: kserve-ci-e2e-test
    resourceVersion: "18407"
    uid: 1df4a5ba-7bf5-468e-8923-219027fb25b9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-141-52.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-141-52.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:00Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69
    namespace: kserve-ci-e2e-test
    resourceVersion: "17923"
    uid: 1f0eb352-794c-496e-8bc7-242f9b507dec
  kind: Event
  lastTimestamp: "2026-09-17T18:12:00Z"
  message: 'Created pod: model-chainer-raw-hpa-7da3c-c8bcbfb69-tddkf'
  metadata:
    creationTimestamp: "2026-09-17T18:12:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:00Z"
    name: model-chainer-raw-hpa-7da3c-c8bcbfb69.18d62e382c98fd08
    namespace: kserve-ci-e2e-test
    resourceVersion: "17928"
    uid: c932bf16-2cc9-4a1b-a9f0-a1fd6a1c18b2
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:00Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: model-chainer-raw-hpa-7da3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17921"
    uid: f8d42ce4-ed41-4335-bb76-3f7692868052
  kind: Event
  lastTimestamp: "2026-09-17T18:12:00Z"
  message: Scaled up replica set model-chainer-raw-hpa-7da3c-c8bcbfb69 from 0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:12:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:12:00Z"
    name: model-chainer-raw-hpa-7da3c.18d62e382bc7d2a5
    namespace: kserve-ci-e2e-test
    resourceVersion: "17924"
    uid: 996e335e-79f2-4841-a78e-83f9568d0b10
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:12:00Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1alpha1
    kind: InferenceGraph
    name: model-chainer-raw-hpa-7da3c
    namespace: kserve-ci-e2e-test
    resourceVersion: "17938"
    uid: 035aa85c-db93-4c32-bc31-a34729e16dc3
  kind: Event
  lastTimestamp: "2026-09-17T18:12:00Z"
  message: InferenceGraph [model-chainer-raw-hpa-7da3c] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:12:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:12:00Z"
    name: model-chainer-raw-hpa-7da3c.18d62e3830fa3f0d
    namespace: kserve-ci-e2e-test
    resourceVersion: "17940"
    uid: f5cbdf22-06fa-41b4-951e-1d5a4b80bcad
  reason: InferenceGraphReady
  reportingComponent: InferenceGraphController
  reportingInstance: ""
  source:
    component: InferenceGraphController
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:13:59.425133Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19245"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    to ip-10-0-134-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:13:59Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:13:59Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e53cf856809
    namespace: kserve-ci-e2e-test
    resourceVersion: "19251"
    uid: e8c8887a-9455-471d-be93-0e4d2c516528
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:59Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:13:59Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-e475b-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:13:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:13:59Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e53d7b3ff66
    namespace: kserve-ci-e2e-test
    resourceVersion: "19260"
    uid: a8068318-0e23-48d2-adb9-6f56b3e4adaa
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:00Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19255"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:00Z"
  message: Add eth0 [10.134.0.34/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:14:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:14:00Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e5416214041
    namespace: kserve-ci-e2e-test
    resourceVersion: "19273"
    uid: 61358d0e-92d6-4386-bafe-c6a2df2b51ee
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:00Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:14:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:00Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e5417cd05a9
    namespace: kserve-ci-e2e-test
    resourceVersion: "19275"
    uid: c13a5099-e58c-404c-ae99-d92dbf2ef83d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:00Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:14:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:00Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e541b3ee6fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "19276"
    uid: 7e56e993-2b2e-492c-991c-c98b0866b122
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:00Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:00Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:14:00Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:00Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e541bc393d1
    namespace: kserve-ci-e2e-test
    resourceVersion: "19277"
    uid: 602d67f4-a355-4b07-8794-a2052a96a678
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:04Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:14:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:04Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e54fcd758e6
    namespace: kserve-ci-e2e-test
    resourceVersion: "19315"
    uid: 55c23a4f-ae8b-45f2-82bd-77c2faa07c6d
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:04Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:14:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:04Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e550020cc8f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19319"
    uid: 854c9e8e-fd03-4a5f-b95c-9b817a66aa8e
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:04Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:14:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:04Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e550098e995
    namespace: kserve-ci-e2e-test
    resourceVersion: "19320"
    uid: 0ff72aa2-99a7-4f25-94e4-caf2b661c485
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:04Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:14:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:04Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e5500a207e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19321"
    uid: b9880b1d-9bca-46a3-8248-1e4a9c6e12dc
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:04Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:14:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:04Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e5503a4c8e4
    namespace: kserve-ci-e2e-test
    resourceVersion: "19323"
    uid: 7daeeded-afab-4bfa-9b64-636c7b29e4fc
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:14:04Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:14:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:14:04Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e55041b6e91
    namespace: kserve-ci-e2e-test
    resourceVersion: "19324"
    uid: dbf02c20-7968-4492-980f-c25f9f264313
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:05Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:15:31Z"
  message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:14:05Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:31Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e5538df0f13
    namespace: kserve-ci-e2e-test
    resourceVersion: "20100"
    uid: 577d2d37-6c79-43fc-b7e7-f4fb7e316748
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:15:29Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:15:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:29Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e68cab124e2
    namespace: kserve-ci-e2e-test
    resourceVersion: "20084"
    uid: f2dfa8b5-67ee-4194-a071-d049d3235c26
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:29Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:15:29Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:15:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:29Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e68cab18be0
    namespace: kserve-ci-e2e-test
    resourceVersion: "20085"
    uid: 8e734bf6-264c-4589-aeb7-59ec90a25cb5
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:31Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl
    namespace: kserve-ci-e2e-test
    resourceVersion: "19247"
    uid: 779249c2-f1b9-4459-ad1a-836fe924c2b5
  kind: Event
  lastTimestamp: "2026-09-17T18:15:31Z"
  message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp
    10.134.0.34:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:15:31Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:31Z"
    name: raw-sklearn-e475b-predictor-5854946c7b-f9lbl.18d62e693f1bfb59
    namespace: kserve-ci-e2e-test
    resourceVersion: "20099"
    uid: e97c3eac-03b5-4069-a495-58a82629ee84
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-134-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-134-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:59Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-e475b-predictor-5854946c7b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19236"
    uid: a608d1e2-4152-4c30-bf09-d6d688a02b5a
  kind: Event
  lastTimestamp: "2026-09-17T18:13:59Z"
  message: 'Created pod: raw-sklearn-e475b-predictor-5854946c7b-f9lbl'
  metadata:
    creationTimestamp: "2026-09-17T18:13:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:59Z"
    name: raw-sklearn-e475b-predictor-5854946c7b.18d62e53cefa3c40
    namespace: kserve-ci-e2e-test
    resourceVersion: "19246"
    uid: 0c9987bc-49ca-4b6b-b2aa-65c61373307d
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:59Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-e475b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19234"
    uid: 6482fd12-6d9a-47e6-afac-5efc3997579e
  kind: Event
  lastTimestamp: "2026-09-17T18:13:59Z"
  message: Scaled up replica set raw-sklearn-e475b-predictor-5854946c7b from 0 to
    1
  metadata:
    creationTimestamp: "2026-09-17T18:13:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:13:59Z"
    name: raw-sklearn-e475b-predictor.18d62e53cc746ecb
    namespace: kserve-ci-e2e-test
    resourceVersion: "19238"
    uid: 9c571144-25bf-4a80-a55e-b55f5ea4a0bd
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:14Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-e475b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19235"
    uid: fbb50e75-fa6b-499d-9187-355f99e7a377
  kind: Event
  lastTimestamp: "2026-09-17T18:14:14Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:14:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:14Z"
    name: raw-sklearn-e475b-predictor.18d62e574b063a2f
    namespace: kserve-ci-e2e-test
    resourceVersion: "19497"
    uid: ba34f359-e0e3-4bf2-bdbb-62cac6d86e6e
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:14Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-e475b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19235"
    uid: fbb50e75-fa6b-499d-9187-355f99e7a377
  kind: Event
  lastTimestamp: "2026-09-17T18:14:14Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:14:14Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:14:14Z"
    name: raw-sklearn-e475b-predictor.18d62e574b6bfba3
    namespace: kserve-ci-e2e-test
    resourceVersion: "19499"
    uid: c4152fa2-224c-4aa0-8615-d7f88a9fdc7b
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:29Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-e475b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19498"
    uid: fbb50e75-fa6b-499d-9187-355f99e7a377
  kind: Event
  lastTimestamp: "2026-09-17T18:15:29Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:14:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:15:29Z"
    name: raw-sklearn-e475b-predictor.18d62e5aca6b8829
    namespace: kserve-ci-e2e-test
    resourceVersion: "20068"
    uid: fbb50e9a-89aa-443c-8634-3f3eea52b22d
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 5
  eventTime: null
  firstTimestamp: "2026-09-17T18:14:29Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-e475b-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "19498"
    uid: fbb50e75-fa6b-499d-9187-355f99e7a377
  kind: Event
  lastTimestamp: "2026-09-17T18:15:29Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:14:29Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:15:29Z"
    name: raw-sklearn-e475b-predictor.18d62e5acab70954
    namespace: kserve-ci-e2e-test
    resourceVersion: "20069"
    uid: db23bc00-0c99-495c-aa88-f1f88a5677e9
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 11
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:59Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-e475b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19227"
    uid: 02da38f1-2009-4c9a-80c8-cf0b90080b94
  kind: Event
  lastTimestamp: "2026-09-17T18:15:21Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:13:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:15:21Z"
    name: raw-sklearn-e475b.18d62e53cc72f9d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20013"
    uid: bc64c90f-edcb-4b2f-a474-4e50f3c78b7a
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:13:59Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-e475b
    namespace: kserve-ci-e2e-test
    resourceVersion: "19227"
    uid: 02da38f1-2009-4c9a-80c8-cf0b90080b94
  kind: Event
  lastTimestamp: "2026-09-17T18:13:59Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-e475b": Operation
    cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-e475b":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:13:59Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:13:59Z"
    name: raw-sklearn-e475b.18d62e53cd015434
    namespace: kserve-ci-e2e-test
    resourceVersion: "19242"
    uid: 6b71e642-c204-4f25-bb9a-1adca5f81259
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:21Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-e475b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20010"
    uid: 02da38f1-2009-4c9a-80c8-cf0b90080b94
  kind: Event
  lastTimestamp: "2026-09-17T18:15:21Z"
  message: InferenceService [raw-sklearn-e475b] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:15:21Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:15:21Z"
    name: raw-sklearn-e475b.18d62e66f03786fb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20012"
    uid: 03ee410e-0ce0-4387-9c31-a06baeab032f
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
- action: Binding
  apiVersion: v1
  eventTime: "2026-09-17T18:15:34.719198Z"
  firstTimestamp: null
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20149"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: null
  message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    to ip-10-0-143-209.ec2.internal
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: events.k8s.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:action: {}
        f:eventTime: {}
        f:note: {}
        f:reason: {}
        f:regarding: {}
        f:reportingController: {}
        f:reportingInstance: {}
        f:type: {}
      manager: kube-scheduler
      operation: Update
      time: "2026-09-17T18:15:34Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e69ff7d7458
    namespace: kserve-ci-e2e-test
    resourceVersion: "20153"
    uid: eb4ce61b-ad1b-4005-8ec1-ff3a9d129950
  reason: Scheduled
  reportingComponent: default-scheduler
  reportingInstance: default-scheduler-kube-scheduler-5556894d5b-tmb7p
  source: {}
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:34Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:34Z"
  message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-5bb89-predictor-serving-cert"
    not found'
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:34Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6a090336d4
    namespace: kserve-ci-e2e-test
    resourceVersion: "20165"
    uid: 880a8fb3-7723-489f-b0a2-7824359dbe34
  reason: FailedMount
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:35Z"
  involvedObject:
    apiVersion: v1
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20158"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:35Z"
  message: Add eth0 [10.133.0.40/23] from ovn-kubernetes
  metadata:
    creationTimestamp: "2026-09-17T18:15:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: multus-daemon
      operation: Update
      time: "2026-09-17T18:15:35Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6a4516b4bb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20177"
    uid: b9883295-c593-4244-a491-f940bc579976
  reason: AddedInterface
  reportingComponent: multus
  reportingInstance: ""
  source:
    component: multus
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:35Z"
  message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd807572e76a5c04b5e09287f537df3a5a34e8aea047c82cf332b6328ae06423"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:15:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:35Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6a46920944
    namespace: kserve-ci-e2e-test
    resourceVersion: "20179"
    uid: 472e1550-e175-4c8d-8aac-9620749c2357
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:35Z"
  message: 'Created container: storage-initializer'
  metadata:
    creationTimestamp: "2026-09-17T18:15:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:35Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6a49cd2796
    namespace: kserve-ci-e2e-test
    resourceVersion: "20181"
    uid: f59be871-30c6-4dc2-ad79-e57a0b48c38f
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:35Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.initContainers{storage-initializer}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:35Z"
  message: Started container storage-initializer
  metadata:
    creationTimestamp: "2026-09-17T18:15:35Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:35Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6a4a3b5292
    namespace: kserve-ci-e2e-test
    resourceVersion: "20183"
    uid: 4d1e5916-fc03-481f-8014-f15f578a2c83
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:40Z"
  message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1919"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:15:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:40Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6b403be844
    namespace: kserve-ci-e2e-test
    resourceVersion: "20224"
    uid: b69612fd-0f38-4d8e-93a9-d72d77e4b8c4
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:40Z"
  message: 'Created container: kserve-container'
  metadata:
    creationTimestamp: "2026-09-17T18:15:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:40Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6b43eaca51
    namespace: kserve-ci-e2e-test
    resourceVersion: "20228"
    uid: 4a5e5904-22ff-4b0f-85db-ae70be987e06
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:40Z"
  message: Started container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:15:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:40Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6b445e352b
    namespace: kserve-ci-e2e-test
    resourceVersion: "20229"
    uid: d9840073-8f3a-4ae0-a527-7f891129d927
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:40Z"
  message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3"
    already present on machine
  metadata:
    creationTimestamp: "2026-09-17T18:15:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:40Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6b4468e34a
    namespace: kserve-ci-e2e-test
    resourceVersion: "20230"
    uid: 904260ba-fc53-4058-bda2-5dd553be030a
  reason: Pulled
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:40Z"
  message: 'Created container: kube-rbac-proxy'
  metadata:
    creationTimestamp: "2026-09-17T18:15:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:40Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6b4798d650
    namespace: kserve-ci-e2e-test
    resourceVersion: "20232"
    uid: e1196eb3-47ed-4e18-89c9-32017a307af4
  reason: Created
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:40Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:15:40Z"
  message: Started container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:15:40Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:15:40Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6b480a3550
    namespace: kserve-ci-e2e-test
    resourceVersion: "20233"
    uid: 0a888dae-7ac4-4422-8f36-3006dd033d99
  reason: Started
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 10
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:42Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:16:52Z"
  message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection
    refused'
  metadata:
    creationTimestamp: "2026-09-17T18:15:42Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:16:52Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e6bb836c891
    namespace: kserve-ci-e2e-test
    resourceVersion: "20765"
    uid: f0915579-03bf-4e1a-b7ee-72d1f4c9b9d9
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kserve-container}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:17:04Z"
  message: Stopping container kserve-container
  metadata:
    creationTimestamp: "2026-09-17T18:17:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:04Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e7ef751eabe
    namespace: kserve-ci-e2e-test
    resourceVersion: "20992"
    uid: 4379569c-fca0-4c02-a91f-3d9461c84436
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:04Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:17:04Z"
  message: Stopping container kube-rbac-proxy
  metadata:
    creationTimestamp: "2026-09-17T18:17:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:04Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e7ef75292dc
    namespace: kserve-ci-e2e-test
    resourceVersion: "20994"
    uid: 06ad4ee8-87e1-4c67-9dde-f3d40b1d7dcc
  reason: Killing
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:08Z"
  involvedObject:
    apiVersion: v1
    fieldPath: spec.containers{kube-rbac-proxy}
    kind: Pod
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs
    namespace: kserve-ci-e2e-test
    resourceVersion: "20151"
    uid: 4bdd3bc6-ac63-4c20-b85a-c0c9f43f5d3d
  kind: Event
  lastTimestamp: "2026-09-17T18:17:08Z"
  message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp
    10.133.0.40:8643: connect: connection refused'
  metadata:
    creationTimestamp: "2026-09-17T18:17:08Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:reportingInstance: {}
        f:source:
          f:component: {}
          f:host: {}
        f:type: {}
      manager: kubelet
      operation: Update
      time: "2026-09-17T18:17:08Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs.18d62e7fbe75b305
    namespace: kserve-ci-e2e-test
    resourceVersion: "21045"
    uid: 860ef438-dc03-4401-b5e9-d2d6c494cb4b
  reason: Unhealthy
  reportingComponent: kubelet
  reportingInstance: ip-10-0-143-209.ec2.internal
  source:
    component: kubelet
    host: ip-10-0-143-209.ec2.internal
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:34Z"
  involvedObject:
    apiVersion: apps/v1
    kind: ReplicaSet
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb
    namespace: kserve-ci-e2e-test
    resourceVersion: "20138"
    uid: 18138af5-f113-4580-ace0-b48d4e69d795
  kind: Event
  lastTimestamp: "2026-09-17T18:15:34Z"
  message: 'Created pod: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb-s4zzs'
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:15:34Z"
    name: raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb.18d62e69feddb7d3
    namespace: kserve-ci-e2e-test
    resourceVersion: "20150"
    uid: 660e72b9-e462-405e-9fc6-cd724ffe7a2c
  reason: SuccessfulCreate
  reportingComponent: replicaset-controller
  reportingInstance: ""
  source:
    component: replicaset-controller
  type: Normal
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:34Z"
  involvedObject:
    apiVersion: apps/v1
    kind: Deployment
    name: raw-sklearn-runtime-5bb89-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20136"
    uid: 3e3f682c-7f62-485b-9ce3-712dffb3e0c2
  kind: Event
  lastTimestamp: "2026-09-17T18:15:34Z"
  message: Scaled up replica set raw-sklearn-runtime-5bb89-predictor-6dddf4b9cb from
    0 to 1
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:15:34Z"
    name: raw-sklearn-runtime-5bb89-predictor.18d62e69fb815f8d
    namespace: kserve-ci-e2e-test
    resourceVersion: "20141"
    uid: 08ca7677-d164-4996-a8b1-64b4842802ce
  reason: ScalingReplicaSet
  reportingComponent: deployment-controller
  reportingInstance: ""
  source:
    component: deployment-controller
  type: Normal
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:49Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5bb89-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20137"
    uid: d8dae38b-8757-4b95-b322-c7c700c9de23
  kind: Event
  lastTimestamp: "2026-09-17T18:16:19Z"
  message: 'failed to get cpu utilization: unable to get metrics for resource cpu:
    no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:15:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:16:19Z"
    name: raw-sklearn-runtime-5bb89-predictor.18d62e6d7a017c6c
    namespace: kserve-ci-e2e-test
    resourceVersion: "20518"
    uid: 50354899-01b9-4c9d-bb45-bacdd389083a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:49Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5bb89-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20137"
    uid: d8dae38b-8757-4b95-b322-c7c700c9de23
  kind: Event
  lastTimestamp: "2026-09-17T18:16:19Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: unable to get metrics for
    resource cpu: no metrics returned from resource metrics API'
  metadata:
    creationTimestamp: "2026-09-17T18:15:49Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:16:19Z"
    name: raw-sklearn-runtime-5bb89-predictor.18d62e6d7a6b7296
    namespace: kserve-ci-e2e-test
    resourceVersion: "20519"
    uid: 92f22c03-8356-409a-ba34-1785efd44736
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:16:04Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5bb89-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20302"
    uid: d8dae38b-8757-4b95-b322-c7c700c9de23
  kind: Event
  lastTimestamp: "2026-09-17T18:16:49Z"
  message: 'failed to get cpu utilization: did not receive metrics for targeted pods
    (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:16:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:16:49Z"
    name: raw-sklearn-runtime-5bb89-predictor.18d62e70f9e793cd
    namespace: kserve-ci-e2e-test
    resourceVersion: "20742"
    uid: 26c9b36c-2216-4356-8659-4e451302cb2a
  reason: FailedGetResourceMetric
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 3
  eventTime: null
  firstTimestamp: "2026-09-17T18:16:04Z"
  involvedObject:
    apiVersion: autoscaling/v2
    kind: HorizontalPodAutoscaler
    name: raw-sklearn-runtime-5bb89-predictor
    namespace: kserve-ci-e2e-test
    resourceVersion: "20302"
    uid: d8dae38b-8757-4b95-b322-c7c700c9de23
  kind: Event
  lastTimestamp: "2026-09-17T18:16:49Z"
  message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu
    resource metric value: failed to get cpu utilization: did not receive metrics
    for targeted pods (pods might be unready)'
  metadata:
    creationTimestamp: "2026-09-17T18:16:04Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: kube-controller-manager
      operation: Update
      time: "2026-09-17T18:16:49Z"
    name: raw-sklearn-runtime-5bb89-predictor.18d62e70fa578c90
    namespace: kserve-ci-e2e-test
    resourceVersion: "20743"
    uid: 05a6b5c5-84a1-453b-9467-fdd0ec37d8cd
  reason: FailedComputeMetricsReplicas
  reportingComponent: horizontal-pod-autoscaler
  reportingInstance: ""
  source:
    component: horizontal-pod-autoscaler
  type: Warning
- apiVersion: v1
  count: 12
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:34Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5bb89
    namespace: kserve-ci-e2e-test
    resourceVersion: "20129"
    uid: bec67c20-0868-4b36-9933-cb7571731be6
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: Istio VirtualService CRD not present; VirtualService reconciliation skipped.
    If you do not use Istio, set ingress.disableIstioVirtualHost=true.
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: raw-sklearn-runtime-5bb89.18d62e69fb7a0dee
    namespace: kserve-ci-e2e-test
    resourceVersion: "20938"
    uid: de5888cb-89c0-476c-88ce-96339bd98308
  reason: VirtualServiceCRDNotFound
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 2
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:34Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5bb89
    namespace: kserve-ci-e2e-test
    resourceVersion: "20129"
    uid: bec67c20-0868-4b36-9933-cb7571731be6
  kind: Event
  lastTimestamp: "2026-09-17T18:15:34Z"
  message: 'Failed to update status for InferenceService "raw-sklearn-runtime-5bb89":
    Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-5bb89":
    the object has been modified; please apply your changes to the latest version
    and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:15:34Z"
    name: raw-sklearn-runtime-5bb89.18d62e69fc0f90b5
    namespace: kserve-ci-e2e-test
    resourceVersion: "20159"
    uid: 1a5b26b7-d80c-4ec0-bf33-0342f108a873
  reason: UpdateFailed
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:15:34Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5bb89
    namespace: kserve-ci-e2e-test
    resourceVersion: "20131"
    uid: bec67c20-0868-4b36-9933-cb7571731be6
  kind: Event
  lastTimestamp: "2026-09-17T18:15:34Z"
  message: 'fails to update InferenceService status: Operation cannot be fulfilled
    on inferenceservices.serving.kserve.io "raw-sklearn-runtime-5bb89": the object
    has been modified; please apply your changes to the latest version and try again'
  metadata:
    creationTimestamp: "2026-09-17T18:15:34Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:15:34Z"
    name: raw-sklearn-runtime-5bb89.18d62e6a0109af59
    namespace: kserve-ci-e2e-test
    resourceVersion: "20160"
    uid: 2ac8db22-f868-43e3-8d1c-b103f48f15ce
  reason: InternalError
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Warning
- apiVersion: v1
  count: 1
  eventTime: null
  firstTimestamp: "2026-09-17T18:17:02Z"
  involvedObject:
    apiVersion: serving.kserve.io/v1beta1
    kind: InferenceService
    name: raw-sklearn-runtime-5bb89
    namespace: kserve-ci-e2e-test
    resourceVersion: "20936"
    uid: bec67c20-0868-4b36-9933-cb7571731be6
  kind: Event
  lastTimestamp: "2026-09-17T18:17:02Z"
  message: InferenceService [raw-sklearn-runtime-5bb89] is Ready
  metadata:
    creationTimestamp: "2026-09-17T18:17:02Z"
    managedFields:
    - apiVersion: v1
      fieldsType: FieldsV1
      fieldsV1:
        f:count: {}
        f:firstTimestamp: {}
        f:involvedObject: {}
        f:lastTimestamp: {}
        f:message: {}
        f:reason: {}
        f:reportingComponent: {}
        f:source:
          f:component: {}
        f:type: {}
      manager: manager
      operation: Update
      time: "2026-09-17T18:17:02Z"
    name: raw-sklearn-runtime-5bb89.18d62e7e798615fe
    namespace: kserve-ci-e2e-test
    resourceVersion: "20937"
    uid: eeffe6bc-6c81-4453-9c2c-0ac22bb2f9c0
  reason: InferenceServiceReady
  reportingComponent: v1beta1Controllers
  reportingInstance: ""
  source:
    component: v1beta1Controllers
  type: Normal
kind: EventList
metadata:
  resourceVersion: "24527"