--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:01:09.966345Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-7bec4-7d868b65cf-hg9nb to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:01:09Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be3289cff82b2f namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: b2caf2b1-cfd0-43ac-9973-c3b5c844b7a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-7bec4-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:10Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be3289d92d5102 namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: ef867f42-80fc-44f8-9f53-ac9cd247ab3a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:10Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:01:10Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be328a0cfc24a0 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 2dc17dbf-f6e0-4fc6-b8f4-9298a7289980 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7bec4} kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:11Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be328a0e4e4565 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: ac8f86c9-2f70-46b1-872d-a29a5de3f7f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7bec4} kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:11Z" message: 'Created container: ensemble-graph-7bec4' metadata: creationTimestamp: "2026-07-01T15:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:11Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be328a116827bf namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 2825ccab-6fe3-450e-b775-bd0bd61d9205 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7bec4} kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:11Z" message: Started container ensemble-graph-7bec4 metadata: creationTimestamp: "2026-07-01T15:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:11Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be328a11d9456a namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 3d84d118-d82d-48e5-8888-61370d9412bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7bec4} kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Stopping container ensemble-graph-7bec4 metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be328c28a18bb3 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: c3828a3d-ba9a-42ec-8f57-1b8cd7188134 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7bec4} kind: Pod name: ensemble-graph-7bec4-7d868b65cf-hg9nb namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bb489ab5-5eea-4f28-a2fc-12328da95cc0 kind: Event lastTimestamp: "2026-07-01T15:01:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:47Z" name: ensemble-graph-7bec4-7d868b65cf-hg9nb.18be328cac7334ed namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: fcffc339-418c-4c93-9331-0240bd0ed2ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-7bec4-7d868b65cf namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: b90d34e2-09e8-4646-b935-a163fa1dc865 kind: Event lastTimestamp: "2026-07-01T15:01:09Z" message: 'Created pod: ensemble-graph-7bec4-7d868b65cf-hg9nb' metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:09Z" name: ensemble-graph-7bec4-7d868b65cf.18be3289cf6cfc26 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: a8003335-9bbc-4e84-917d-3670760d2c9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 58968419-eb70-4c01-9b3b-40847363ec3f kind: Event lastTimestamp: "2026-07-01T15:01:09Z" message: Scaled up replica set ensemble-graph-7bec4-7d868b65cf from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:09Z" name: ensemble-graph-7bec4.18be3289ceac30dd namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 3b1378f4-b58b-4b8c-90c3-151b39f13ed4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: b971bb39-a8b2-4035-b7b1-92ce83663b8f kind: Event lastTimestamp: "2026-07-01T15:01:10Z" message: InferenceGraph [ensemble-graph-7bec4] is Ready metadata: creationTimestamp: "2026-07-01T15:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:10Z" name: ensemble-graph-7bec4.18be3289d3e94808 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 3faf783b-3c01-483c-9bb5-d8a8f1550931 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:02:20.267964Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-9eaf2-f76f65bd7-dlvlw to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be329a2e45d639 namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 0c3f9291-2cf7-4ed5-93de-5f5c40dff03e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be329a47b3b41f namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: 99304297-e101-4dbd-bc89-833c5a0701a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9eaf2} kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be329a48f4dd38 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 58ee7a28-a3fb-4898-84bf-13dde12eee99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9eaf2} kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Created container: ensemble-graph-9eaf2' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be329a4bdc29a6 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 2eff2ddd-9c90-4426-921d-76f7800a1229 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9eaf2} kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Started container ensemble-graph-9eaf2 metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be329a4c4ae5c4 namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: d987e0f0-0bb7-402c-9fcb-9cbff4427b59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9eaf2} kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: "2026-07-01T15:10:34Z" message: Stopping container ensemble-graph-9eaf2 metadata: creationTimestamp: "2026-07-01T15:10:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:34Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be330d593f767e namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: 80c5fdc0-d419-491f-9154-8edd64e69297 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9eaf2} kind: Pod name: ensemble-graph-9eaf2-f76f65bd7-dlvlw namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 0809d1a8-c26f-474f-a311-2175f16753c0 kind: Event lastTimestamp: "2026-07-01T15:11:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:02Z" name: ensemble-graph-9eaf2-f76f65bd7-dlvlw.18be330df3c4f925 namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: a65f0c3e-1071-430d-877e-6c987b2629bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-9eaf2-f76f65bd7 namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 850b8a9d-0527-4941-a97f-3152ac1e904c kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Created pod: ensemble-graph-9eaf2-f76f65bd7-dlvlw' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2-f76f65bd7.18be329a2dd50a46 namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: d5df088b-d216-4304-bb8c-ee8bc5f783f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: b5ad4b5e-1fe4-4069-af57-7c7e2e996560 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Scaled up replica set ensemble-graph-9eaf2-f76f65bd7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2.18be329a2d1990fb namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: ae024e76-0cd8-4246-9462-32e32b6f6e30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 577d60db-0762-4d0b-9261-a6e427305c2a kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: InferenceGraph [ensemble-graph-9eaf2] is Ready metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:20Z" name: ensemble-graph-9eaf2.18be329a32d81cf3 namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 59494c09-672e-4300-a7be-98ec230ef4e8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:02:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: 3de989ca-8b81-448d-aac3-2b848dc46d7a kind: Event lastTimestamp: "2026-07-01T15:02:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:50Z" name: ensemble-graph-9eaf2.18be329daba47803 namespace: kserve-ci-e2e-test resourceVersion: "22191" uid: 51ab5eff-3bb1-4db3-a596-0ff2de31c3b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:02:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: 3de989ca-8b81-448d-aac3-2b848dc46d7a kind: Event lastTimestamp: "2026-07-01T15:02:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:50Z" name: ensemble-graph-9eaf2.18be329dac076805 namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: d9ba1408-6f1c-4ca1-bb8b-00c23396856b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:51:35.081243Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:51:35Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be3203f625b5be namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: bd635c63-f55f-41e5-a2d2-e40b75d30651 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0d08f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:35Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be3204018b7215 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: eea29191-439e-4e62-b0ff-d1d2889ce516 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be320433ff5443 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: c4839a3b-4196-4120-8146-1278b689b39c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32043592dc2c namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: e4759a1c-3804-44b3-9736-378e4435a946 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be320438af5d26 namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: ff832704-ab41-4c48-b59a-66d7e067b18d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be3204392c4eb2 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: b4d141e6-84c3-4f8c-9631-3d2521743715 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32043937cc2f namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 72342b9c-2cfd-43be-85d6-17e67e8da12a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32043de288f3 namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: f0dffcf3-9c5e-4943-858d-22f782c09df8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32043e5b044e namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 42196b0e-9093-40a5-9135-beea28fc9f8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:51:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T14:52:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:13Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32047db9a18e namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 91f4d80a-86d8-433d-b0c8-cdaeb7d426c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32877a823fae namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 4ecd3477-3ad3-4396-b480-428f4c24c375 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 407379a1-739e-4c98-8bc7-8e2a23fb15e1 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5.18be32877a82f90b namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: f4af4002-0cd9-472e-8be9-8326067d5981 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0d08f-predictor-7fc6f58f4d namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: 9e330802-d306-4810-a148-2b209c2f6276 kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: 'Created pod: error-404-isvc-0d08f-predictor-7fc6f58f4d-gx2v5' metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:35Z" name: error-404-isvc-0d08f-predictor-7fc6f58f4d.18be3203f58f6bdf namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 22f36313-3609-439f-9fd3-1f256ac54dcd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 03eb5ac8-0406-40b7-bbac-d8e63c0a409c kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: Scaled up replica set error-404-isvc-0d08f-predictor-7fc6f58f4d from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:35Z" name: error-404-isvc-0d08f-predictor.18be3203f3d615a8 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 93e6e753-34f0-429e-8ecf-40383a4bd6d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:51:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 9b4b2a62-c843-40d0-bc50-07c44fd1c7e7 kind: Event lastTimestamp: "2026-07-01T14:52:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:35Z" name: error-404-isvc-0d08f-predictor.18be320772682daf namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: a2534691-9183-44a8-a3be-3e9a1b02d502 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:51:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 9b4b2a62-c843-40d0-bc50-07c44fd1c7e7 kind: Event lastTimestamp: "2026-07-01T14:52:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:35Z" name: error-404-isvc-0d08f-predictor.18be320772d9e1c9 namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 38cb18bb-08be-4083-8567-f6da463a7a29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: 9b4b2a62-c843-40d0-bc50-07c44fd1c7e7 kind: Event lastTimestamp: "2026-07-01T14:52:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:05Z" name: error-404-isvc-0d08f-predictor.18be320af1959710 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 38f6b859-5b32-4941-9ead-bb7784f06cad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: 9b4b2a62-c843-40d0-bc50-07c44fd1c7e7 kind: Event lastTimestamp: "2026-07-01T14:52:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:05Z" name: error-404-isvc-0d08f-predictor.18be320af1fd8ddc namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 05eccbed-2ca7-4905-a709-fcc81fd11e43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 3e354411-7796-4a0d-9410-a206b11db51a kind: Event lastTimestamp: "2026-07-01T14:52:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:23Z" name: error-404-isvc-0d08f.18be3203f3dbdbe5 namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: cf65292b-dd83-439b-9b02-49edcdb70256 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 3e354411-7796-4a0d-9410-a206b11db51a kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-0d08f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0d08f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:35Z" name: error-404-isvc-0d08f.18be3203f45cfa98 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 8c237740-2933-4a24-a975-6b8ef1adc6e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 3e354411-7796-4a0d-9410-a206b11db51a kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0d08f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:35Z" name: error-404-isvc-0d08f.18be3203f45d6288 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 0dec8f1d-92b3-4757-b5e7-82678fbf5805 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: 3e354411-7796-4a0d-9410-a206b11db51a kind: Event lastTimestamp: "2026-07-01T14:52:23Z" message: InferenceService [error-404-isvc-0d08f] is Ready metadata: creationTimestamp: "2026-07-01T14:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:23Z" name: error-404-isvc-0d08f.18be320f396e441e namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: f30bc034-c7fe-47a1-a69a-8a5d3487e6be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:00:59.896816Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-102b8-predictor-548dfbff4-mm86m to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be328777c75900 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: df0f66f9-46fd-4ab5-9af5-d217933c7bde reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-102b8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be328784b7c172 namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 37313533-95e7-4be2-be52-72208e7787d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:01:00Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287b5a956a2 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 388f61c1-b645-4b49-ba52-0fe575e96c48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287b72303f8 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 74a0b5b3-dfe3-40ba-86ff-680442bc9d17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:01Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287ba6641b2 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 67965310-7491-4bd8-800c-4ad9f1bc0ef1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:01Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287badeca2a namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 3d576265-1a3d-4678-af23-ac289253dcf7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:01Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287bae900e2 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 9543ed57-d932-4d73-88c1-682441a24544 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:01Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287be91b2bd namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 40b4b357-73f5-4d0b-ae6d-edda520c08a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:01Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3287bf22e8c9 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 4af93225-d892-45ec-8c45-9757c08b7976 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:01:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:01:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:39Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be3288348d769f namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 1b33eda4-03b9-4dbc-804b-7e41c15d96f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be329a271b0c3a namespace: kserve-ci-e2e-test resourceVersion: "21848" uid: 7873df37-23a0-4eb5-9204-a985970fd267 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-102b8-predictor-548dfbff4-mm86m namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 0376aec0-5160-4084-8722-f11dd701e097 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-102b8-predictor-548dfbff4-mm86m.18be329a271bb6a9 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 858ae3a6-d2d8-4fa2-b4ca-c4f4ece5f261 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-102b8-predictor-548dfbff4 namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 95b24e19-eeea-4ab2-985d-f868f8cef641 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: 'Created pod: error-404-isvc-102b8-predictor-548dfbff4-mm86m' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-102b8-predictor-548dfbff4.18be32877749ff4e namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: 40e73682-eb2a-49da-ae10-01e6856840c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: f785edf5-5ee4-4719-98b1-5929839caf57 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Scaled up replica set error-404-isvc-102b8-predictor-548dfbff4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-102b8-predictor.18be328775c6f681 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 50c471e1-7a8a-4f5b-bfff-070993fa4a76 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: a88d1da6-f8e7-499b-8a28-9d700693dd28 kind: Event lastTimestamp: "2026-07-01T15:01:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:29Z" name: error-404-isvc-102b8-predictor.18be328af421c964 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 33eb413c-dea5-4310-954d-7f699bfc74e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: a88d1da6-f8e7-499b-8a28-9d700693dd28 kind: Event lastTimestamp: "2026-07-01T15:01:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:29Z" name: error-404-isvc-102b8-predictor.18be328af46c990a namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: f888f000-471b-44d4-9b02-66b496fdf531 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:01:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: a88d1da6-f8e7-499b-8a28-9d700693dd28 kind: Event lastTimestamp: "2026-07-01T15:02:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:14Z" name: error-404-isvc-102b8-predictor.18be3291f1f789aa namespace: kserve-ci-e2e-test resourceVersion: "21716" uid: 19bcd483-073e-43e5-84ae-d1ac5faec794 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:01:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: a88d1da6-f8e7-499b-8a28-9d700693dd28 kind: Event lastTimestamp: "2026-07-01T15:02:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:14Z" name: error-404-isvc-102b8-predictor.18be3291f26bacdc namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: 39168f4d-af57-47e9-afd9-2edfb0bd3b9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: d778e8b2-631a-4122-b031-7d155611b86f kind: Event lastTimestamp: "2026-07-01T15:01:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:49Z" name: error-404-isvc-102b8.18be328775cd1863 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: df67c62a-ee8a-4f3c-928a-563722d07509 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: d778e8b2-631a-4122-b031-7d155611b86f kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-102b8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-102b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-102b8.18be3287764d4a65 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 1e3ca33f-697b-4983-87e7-283e26c7649e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: d778e8b2-631a-4122-b031-7d155611b86f kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-102b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:59Z" name: error-404-isvc-102b8.18be3287764d89c7 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 7e2a9bbf-841c-4ecf-a9f2-75d593120619 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: d778e8b2-631a-4122-b031-7d155611b86f kind: Event lastTimestamp: "2026-07-01T15:01:49Z" message: InferenceService [error-404-isvc-102b8] is Ready metadata: creationTimestamp: "2026-07-01T15:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:49Z" name: error-404-isvc-102b8.18be3292efd0b732 namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 83248d8c-078c-4cd4-bed9-d8b582e0171b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:11:45.697284Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-23f07-predictor-5c87794f48-svfqr to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331dd47c7d57 namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 57201c15-13ea-41a5-b19f-0e64e313c8a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331deed63704 namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: 99929c7f-7d63-424f-a9f2-c7887816073e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331df10179fa namespace: kserve-ci-e2e-test resourceVersion: "26703" uid: 1bc1e479-09c7-46a8-b4ed-a240228629b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331df43fac7b namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: 2c7167f4-dba6-4ba2-b9ca-3a6c7a1e02f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331df4b901ef namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: e1bdebd2-913f-4ebe-b49b-f72ce2aaac25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331df4c489fd namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: 0155c9b2-dede-4d9c-b52e-1843f8d35b57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331df79ae385 namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 0adf2201-f459-4050-82a8-b331c23626de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331df816d95f namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: f09d0ed5-a168-4c8a-b1a2-204d29304e8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:21:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:13Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be331e2860082e namespace: kserve-ci-e2e-test resourceVersion: "30918" uid: db8c2a62-5e4b-4a86-b44e-b79a741605cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:21:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:10Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be33a148b71131 namespace: kserve-ci-e2e-test resourceVersion: "30896" uid: 4012562c-100a-442a-80e6-d7bbaf14d18b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:21:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:10Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be33a148b784bb namespace: kserve-ci-e2e-test resourceVersion: "30897" uid: cd41984f-741c-47aa-acd2-b88589863e7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:21:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23f07-predictor-5c87794f48-svfqr namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: ac82ba03-c329-4df9-9e6e-fa75a096fa4b kind: Event lastTimestamp: "2026-07-01T15:21:13Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:13Z" name: error-404-isvc-23f07-predictor-5c87794f48-svfqr.18be33a1f0d0bb8a namespace: kserve-ci-e2e-test resourceVersion: "30917" uid: 6f583a52-3310-4d1b-97aa-a5ae58921e10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-23f07-predictor-5c87794f48 namespace: kserve-ci-e2e-test resourceVersion: "26659" uid: ceda14a3-6877-4887-a186-ff594dabc425 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Created pod: error-404-isvc-23f07-predictor-5c87794f48-svfqr' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-23f07-predictor-5c87794f48.18be331dd3f27000 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: cd798e9f-a6f7-40d1-adf0-e8db0833c1d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26657" uid: d0a98d5c-7edb-4cc3-b29d-5d98eab9d195 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Scaled up replica set error-404-isvc-23f07-predictor-5c87794f48 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-23f07-predictor.18be331dd20f1cf9 namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: 8b1c6309-5b64-4e73-9d74-e5f7ca8cf674 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: 6e8d88d2-dd09-41a0-8823-73f8136ef3ae kind: Event lastTimestamp: "2026-07-01T15:12:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:15Z" name: error-404-isvc-23f07-predictor.18be33215072d16a namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 7a06b1bb-cd33-4c5e-8473-902ec6a47a0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: 6e8d88d2-dd09-41a0-8823-73f8136ef3ae kind: Event lastTimestamp: "2026-07-01T15:12:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:15Z" name: error-404-isvc-23f07-predictor.18be332150dadf23 namespace: kserve-ci-e2e-test resourceVersion: "27013" uid: 60af8e4f-137e-42b4-bfbe-52210c04d285 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: 6e8d88d2-dd09-41a0-8823-73f8136ef3ae kind: Event lastTimestamp: "2026-07-01T15:13:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:00Z" name: error-404-isvc-23f07-predictor.18be33284e540efb namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: b58b9157-b190-45d4-bc85-2c77341d8da2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: 6e8d88d2-dd09-41a0-8823-73f8136ef3ae kind: Event lastTimestamp: "2026-07-01T15:13:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:00Z" name: error-404-isvc-23f07-predictor.18be33284ebec8c8 namespace: kserve-ci-e2e-test resourceVersion: "27456" uid: 9941b1c1-0cef-4e18-b357-4d03e74e0f9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23f07 namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 368e59c9-284e-47c8-bd38-a62fd504f475 kind: Event lastTimestamp: "2026-07-01T15:12:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:33Z" name: error-404-isvc-23f07.18be331dd21dc76b namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: 00690406-a984-4f6d-bc73-68fad4f7417e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 368e59c9-284e-47c8-bd38-a62fd504f475 kind: Event lastTimestamp: "2026-07-01T15:12:33Z" message: InferenceService [error-404-isvc-23f07] is Ready metadata: creationTimestamp: "2026-07-01T15:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:33Z" name: error-404-isvc-23f07.18be3328e69134ca namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: eb9b74bd-ebf5-4c48-ab50-248e7ffdfdc7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:50:45.130777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3297d-predictor-647cb88f78-7q4rr to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f854de1dcd namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: e37722e7-d604-4008-a9fc-8a4b397d90f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f86ec34b7f namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 5c8a2702-c676-4154-b754-f5d557cf3380 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f87037b5ff namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 4606650d-7f5a-4349-9b2a-8ebd5dbec908 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f873308e4c namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 63332735-e093-4784-88bc-d3a70acf1731 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f873be0a99 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 82ab616e-3b58-40f2-8c20-8055fbfa8ab5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f873c9d308 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: e12f3c19-df6e-423e-83cb-0ba37f287776 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f87661b669 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 6a1e0ef4-930b-4421-825c-19d467c4f878 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f876dacaa7 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 9007b209-7ade-4834-a633-3ea74ba272c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T15:00:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:02Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be31f88e15a3f7 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: fd4f1b72-f86e-4e89-8f56-4209b007d909 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be32798b5de1d7 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: bb262733-5ba7-40d2-89c5-83117e18be15 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be32798b5e61b4 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: d6b4a65c-8610-478b-8ffe-ddebedea1f02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: efa66ef8-22f0-4980-b0b8-adba388ec0d1 kind: Event lastTimestamp: "2026-07-01T15:00:02Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:02Z" name: error-404-isvc-3297d-predictor-647cb88f78-7q4rr.18be327a028a749e namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: ae7cee7c-110b-4017-b391-1a37e6e2c5d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3297d-predictor-647cb88f78 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 51256778-d7f7-4c92-801f-518c3d9500e1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: 'Created pod: error-404-isvc-3297d-predictor-647cb88f78-7q4rr' metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor-647cb88f78.18be31f85453409a namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 7bb76443-3836-4205-8a8c-180e0fcc5950 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: ce9f2441-22ab-4e49-b615-f79ef3c1c4ba kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Scaled up replica set error-404-isvc-3297d-predictor-647cb88f78 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d-predictor.18be31f8516438a9 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: fc40b699-a88e-40a5-a2b0-0ebd3d568947 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: c16d2fe4-39df-48ee-96fd-6c959c2742bd kind: Event lastTimestamp: "2026-07-01T14:51:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:15Z" name: error-404-isvc-3297d-predictor.18be31fbcee140f2 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: b7fc9d49-e6ba-47f0-a12a-3f95ef60ac73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: c16d2fe4-39df-48ee-96fd-6c959c2742bd kind: Event lastTimestamp: "2026-07-01T14:51:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:15Z" name: error-404-isvc-3297d-predictor.18be31fbcf56afd7 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 30a27117-323e-450e-a30c-ed78220d8bb8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: c16d2fe4-39df-48ee-96fd-6c959c2742bd kind: Event lastTimestamp: "2026-07-01T14:52:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:15Z" name: error-404-isvc-3297d-predictor.18be3202cce3fe3f namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 9a486206-e84c-4d7c-ba81-3164b6f55f56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: c16d2fe4-39df-48ee-96fd-6c959c2742bd kind: Event lastTimestamp: "2026-07-01T14:52:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:15Z" name: error-404-isvc-3297d-predictor.18be3202cd3dea76 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 79cf26ae-458d-4685-9651-c037af6a70fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: a02e29b4-56b0-4c35-87df-e9a6c1a28a6b kind: Event lastTimestamp: "2026-07-01T14:51:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:32Z" name: error-404-isvc-3297d.18be31f85104ca47 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 4a36a6b4-79f5-4d88-a488-fc6222542436 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: a02e29b4-56b0-4c35-87df-e9a6c1a28a6b kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-3297d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3297d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d.18be31f851a1d68f namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: d9b3f6ac-436d-4300-8080-49b4bc1eaf2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: a02e29b4-56b0-4c35-87df-e9a6c1a28a6b kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3297d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-3297d.18be31f851a22a3c namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: e0f1eaf0-1b75-4bc7-a8a9-50415cb8baf7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: a02e29b4-56b0-4c35-87df-e9a6c1a28a6b kind: Event lastTimestamp: "2026-07-01T14:51:32Z" message: InferenceService [error-404-isvc-3297d] is Ready metadata: creationTimestamp: "2026-07-01T14:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:32Z" name: error-404-isvc-3297d.18be32034ad0847b namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: ce665892-bd4f-408d-b99e-bbab2668589a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:04.956988Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-680c6-predictor-795547b9b6-hjms6 to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:04Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e1020b5a8c namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 455f2e9b-7074-44b8-8512-b5710891a90b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-680c6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e10f7a4d2a namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: f041330d-10d7-4cd7-a7cd-6197be1eb860 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:06Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:06Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e140ad2322 namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 218e32df-fcf5-4a8a-a2ec-6d0236b90b44 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:06Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T14:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:06Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e1421e4163 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 432b33e2-bddb-4386-9a44-cc47c5a29eaf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:25Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" in 19.493s (19.493s including waiting). Image size: 1334624876 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:25Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e5cc089efb namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 3d74fb75-ac30-4e05-be7b-3b9833d82b93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:33Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e7c28f18ed namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 6d587225-06d1-4be9-90fa-7e5a1922375a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:33Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e7c36a7a29 namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 482781bd-1065-455b-9ec3-3df4efd95fa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:34Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e7ca1a432c namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 6e53764a-6ead-4b42-9616-d34809bf175a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.359s (3.359s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e89258d3d6 namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 481f9fd0-ea83-43b5-b1bb-701b0ffcab15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e899fd9df3 namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: bbeecbdd-a307-4c78-99e5-21ed859dac8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e89b4e3bc8 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: b1f02716-0f84-4283-ad40-3b7c42de5963 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:50:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:14Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31e8e0c560b9 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 7a14a812-649e-4522-b17d-4cd2b815402d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31f85af3fb12 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: b85a1dd6-c3ea-4210-9984-dfcdeb0bdc2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-680c6-predictor-795547b9b6-hjms6 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 1a7a1f63-1a0c-40aa-bb5e-2d44e336d230 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: error-404-isvc-680c6-predictor-795547b9b6-hjms6.18be31f85af49a9d namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: d7311aee-cbd9-45db-af3a-779a0ceaeeba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-680c6-predictor-795547b9b6 namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 504469ea-5474-4a4c-a92b-8607ab4caefc kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'Created pod: error-404-isvc-680c6-predictor-795547b9b6-hjms6' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:04Z" name: error-404-isvc-680c6-predictor-795547b9b6.18be31e101924311 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 30fd95f5-ddac-41ac-ab4c-7c90e9d0c74e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 35ad0448-d36f-4198-8a94-514c98651458 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: Scaled up replica set error-404-isvc-680c6-predictor-795547b9b6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:04Z" name: error-404-isvc-680c6-predictor.18be31e0ff7f5bcc namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: dd8dcffd-cf07-47d8-8358-6ea6309c14fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 782583b5-6a96-44af-b8ac-4d1eda384b50 kind: Event lastTimestamp: "2026-07-01T14:50:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:04Z" name: error-404-isvc-680c6-predictor.18be31e47de7cd7f namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: b97fd4b0-164f-45fc-b547-bd57e12a6265 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 782583b5-6a96-44af-b8ac-4d1eda384b50 kind: Event lastTimestamp: "2026-07-01T14:50:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:04Z" name: error-404-isvc-680c6-predictor.18be31e47e59c1f0 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 8466beb0-2965-4253-b1ec-4413a6aa05ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 782583b5-6a96-44af-b8ac-4d1eda384b50 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:35Z" name: error-404-isvc-680c6-predictor.18be31eb7bb3fda2 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: cd81db63-0473-4741-9651-95abd9d569a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 782583b5-6a96-44af-b8ac-4d1eda384b50 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:35Z" name: error-404-isvc-680c6-predictor.18be31eb7c182126 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: ba3a4794-5c22-499f-9f37-2966e67cd101 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-680c6 namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: e6a9b8f9-0773-4798-beb2-5f83f1ac0603 kind: Event lastTimestamp: "2026-07-01T14:50:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:24Z" name: error-404-isvc-680c6.18be31e0ff82a498 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: f3223213-7e37-45ce-b36b-cc78861abbf9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-680c6 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e6a9b8f9-0773-4798-beb2-5f83f1ac0603 kind: Event lastTimestamp: "2026-07-01T14:50:24Z" message: InferenceService [error-404-isvc-680c6] is Ready metadata: creationTimestamp: "2026-07-01T14:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:24Z" name: error-404-isvc-680c6.18be31f39cd51049 namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 5743798d-4696-4181-847e-c964c3950d70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:02:20.164772Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6e167-predictor-747c78f446-zrpn2 to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a281f362a namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: cab395c1-ede1-4267-83ed-b4e5043f03ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6e167-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a36e12cea namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: 9d003afd-1fc8-4f3c-a659-2bd41d9402ec reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a65db96b7 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 27557053-58cb-4ce1-9d57-f36c16ec26c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a67531933 namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 394cef54-2b76-40b5-b1de-7bbbd1d49de5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a6a5f5660 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 596b01ae-aade-4029-862e-21be55be16f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a6ad5e3e4 namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: fdfa6af8-9a4a-4893-92e8-a3cafdfde757 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a6adf7db2 namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: 364900df-61f1-4b5b-9687-863ddd970459 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a6fa8590f namespace: kserve-ci-e2e-test resourceVersion: "21936" uid: 15e744b3-513e-4fdf-836a-fb72d8984105 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329a7045b95a namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: bb135fab-e7b1-4da9-a7ee-2df5748f03cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:02:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:02:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:58Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be329ab05b78a3 namespace: kserve-ci-e2e-test resourceVersion: "22253" uid: 3fdf7e5c-3415-47a6-b4aa-621e3852113b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:11:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:44Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be331daa2027c5 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: 869c24ad-6e06-4854-ae30-619ecf636b2b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e167-predictor-747c78f446-zrpn2 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: cfc49d17-be73-45fa-af42-982df31e15cb kind: Event lastTimestamp: "2026-07-01T15:11:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:44Z" name: error-404-isvc-6e167-predictor-747c78f446-zrpn2.18be331daa20afde namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: 6e38380b-21a4-4d03-b624-771e3b85de24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6e167-predictor-747c78f446 namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: 5b49cb48-51b2-49eb-ab56-67761bacf02e kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Created pod: error-404-isvc-6e167-predictor-747c78f446-zrpn2' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-6e167-predictor-747c78f446.18be329a27b03753 namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: d6cc419b-6223-4104-97c9-09b0428751b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 60c5f076-6e41-4554-9f77-331c88e8d820 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Scaled up replica set error-404-isvc-6e167-predictor-747c78f446 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-6e167-predictor.18be329a25dbfcc5 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 987da351-ffd2-48d6-a8d1-a3c7ebc2fed5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:02:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 32df4044-99f1-44a8-a818-75bf3d9b5b8c kind: Event lastTimestamp: "2026-07-01T15:02:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:50Z" name: error-404-isvc-6e167-predictor.18be329da464daed namespace: kserve-ci-e2e-test resourceVersion: "22188" uid: d452ac10-ee8e-4275-b4a8-2b660ede2046 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:02:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 32df4044-99f1-44a8-a818-75bf3d9b5b8c kind: Event lastTimestamp: "2026-07-01T15:02:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:50Z" name: error-404-isvc-6e167-predictor.18be329da4c7ca50 namespace: kserve-ci-e2e-test resourceVersion: "22189" uid: 0c311795-a688-47c7-ab2d-7ae2e18b1fee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:03:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 32df4044-99f1-44a8-a818-75bf3d9b5b8c kind: Event lastTimestamp: "2026-07-01T15:03:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:20Z" name: error-404-isvc-6e167-predictor.18be32a4a2251cf1 namespace: kserve-ci-e2e-test resourceVersion: "22429" uid: 706a9b3e-625e-4efe-8c44-1564f1ce1930 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:03:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 32df4044-99f1-44a8-a818-75bf3d9b5b8c kind: Event lastTimestamp: "2026-07-01T15:03:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:20Z" name: error-404-isvc-6e167-predictor.18be32a4a2887023 namespace: kserve-ci-e2e-test resourceVersion: "22430" uid: b81b9f00-2dbe-46f3-a2cb-232e71bc453b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "21814" uid: 7d5c0e0c-07e3-4864-b8e0-1da9fe00a1e5 kind: Event lastTimestamp: "2026-07-01T15:03:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:03:08Z" name: error-404-isvc-6e167.18be329a25f0e147 namespace: kserve-ci-e2e-test resourceVersion: "22337" uid: 941b609f-faef-4d8d-acf7-fc6638f8b1f5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "21814" uid: 7d5c0e0c-07e3-4864-b8e0-1da9fe00a1e5 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-6e167": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6e167": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-6e167.18be329a26993e02 namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: 78060de5-042b-4799-b7ac-89a971a8d711 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "21814" uid: 7d5c0e0c-07e3-4864-b8e0-1da9fe00a1e5 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6e167": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-6e167.18be329a2699a407 namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: 53bbc442-a21f-4fff-87be-1052f5e3c1e0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22334" uid: 7d5c0e0c-07e3-4864-b8e0-1da9fe00a1e5 kind: Event lastTimestamp: "2026-07-01T15:03:08Z" message: InferenceService [error-404-isvc-6e167] is Ready metadata: creationTimestamp: "2026-07-01T15:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:03:08Z" name: error-404-isvc-6e167.18be32a56bb7cdc4 namespace: kserve-ci-e2e-test resourceVersion: "22335" uid: 7202d3b3-a74a-440a-be69-cb69244d6f9e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:10:35.180833Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6f42a-predictor-5b8658b774-p5rwr to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330d6960d733 namespace: kserve-ci-e2e-test resourceVersion: "25806" uid: bcb382f4-14fa-44c4-b048-c4875f4b96ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6f42a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330d728d3c68 namespace: kserve-ci-e2e-test resourceVersion: "25837" uid: fbc2f75b-e1a2-4d3d-9b3a-1bab28147914 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330da72b533f namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: f6a343c1-516a-4e88-a561-aa027865a74d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330da8b1ccdb namespace: kserve-ci-e2e-test resourceVersion: "25857" uid: 69833d56-1cb2-47d9-abb4-e88e39eacdfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330dac5b4e37 namespace: kserve-ci-e2e-test resourceVersion: "25859" uid: a1e04f2a-58e3-4886-9232-69c5a6f99a27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330dacd64594 namespace: kserve-ci-e2e-test resourceVersion: "25861" uid: 117a51f9-f537-4ec1-b1b3-9054e9dcc0dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330dace01c89 namespace: kserve-ci-e2e-test resourceVersion: "25862" uid: 6a32a85a-0149-4b2b-8bfe-3e8357214325 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330dafef3275 namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: c86d9fa3-347d-463e-a616-c5a7ca5653d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330db066d1ed namespace: kserve-ci-e2e-test resourceVersion: "25865" uid: 63442404-9597-40c0-b212-2473a06b8504 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:11:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:13Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be330e03abea2b namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 39961a72-6992-4bb7-aab4-3355f5c1bf36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be331dc4c2599d namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: f5b85d80-b619-4118-8fb8-b6e3edb7c135 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be331dc4c2e436 namespace: kserve-ci-e2e-test resourceVersion: "26619" uid: 2dbd7d93-a682-4b9e-bd47-775ece1d5f03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 398874b0-8fd2-4d31-b7b1-3e07e069ce1a kind: Event lastTimestamp: "2026-07-01T15:11:48Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:48Z" name: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr.18be331e8bff421f namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: a3773ff0-f937-4d53-a907-2659d592ed3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6f42a-predictor-5b8658b774 namespace: kserve-ci-e2e-test resourceVersion: "25795" uid: f9004a1f-3de9-4837-b674-46305125c4a9 kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'Created pod: error-404-isvc-6f42a-predictor-5b8658b774-p5rwr' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-6f42a-predictor-5b8658b774.18be330d68fe749a namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: b3708b31-846c-4b13-93bc-dcdc347131f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: 48d5825c-bcdc-4a2f-af62-75cbb05f535b kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: Scaled up replica set error-404-isvc-6f42a-predictor-5b8658b774 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-6f42a-predictor.18be330d6771586f namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: e3dd5ba1-9a3d-4417-a900-2ce7aa03338d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:10:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 0cb83bab-48c5-4bbf-83ae-ddf88abe09df kind: Event lastTimestamp: "2026-07-01T15:11:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:20Z" name: error-404-isvc-6f42a-predictor.18be3310e61e6287 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: 2d6726db-6cda-4499-9372-488a107f96c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:10:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 0cb83bab-48c5-4bbf-83ae-ddf88abe09df kind: Event lastTimestamp: "2026-07-01T15:11:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:20Z" name: error-404-isvc-6f42a-predictor.18be3310e6a781d0 namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: adac587a-afd8-45a8-8dcb-ec94564a4bf0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: 0cb83bab-48c5-4bbf-83ae-ddf88abe09df kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:35Z" name: error-404-isvc-6f42a-predictor.18be331b62b57120 namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: f16a0cbd-9487-478a-9f5b-e263bca96419 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: 0cb83bab-48c5-4bbf-83ae-ddf88abe09df kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:35Z" name: error-404-isvc-6f42a-predictor.18be331b634183e7 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 7dac7478-2c0e-4855-85e6-4fbcd10c422d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: abc46759-057b-4fd4-af5f-3875b23cc363 kind: Event lastTimestamp: "2026-07-01T15:11:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:23Z" name: error-404-isvc-6f42a.18be330d67ae9c3b namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: de1d4a32-6935-48d0-b1a5-f99358562358 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: abc46759-057b-4fd4-af5f-3875b23cc363 kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-6f42a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6f42a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-6f42a.18be330d680be48d namespace: kserve-ci-e2e-test resourceVersion: "25799" uid: 5b8de36e-d397-48f0-953c-42bef969d617 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: abc46759-057b-4fd4-af5f-3875b23cc363 kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6f42a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-6f42a.18be330d680c3070 namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: 890e5702-f654-4331-b66d-d507d8a3115b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: abc46759-057b-4fd4-af5f-3875b23cc363 kind: Event lastTimestamp: "2026-07-01T15:11:23Z" message: InferenceService [error-404-isvc-6f42a] is Ready metadata: creationTimestamp: "2026-07-01T15:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:23Z" name: error-404-isvc-6f42a.18be3318bef8cc43 namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: 2f50f4fb-90ac-4af9-a165-82fd2cb63028 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:00:00.053192Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be327988d23032 namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 0a201f8e-7eb3-497b-96da-9b1e7d6b63bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279a2e1e1fb namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 305cbbc9-6dbc-4160-a795-4a729050e815 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279a45dfdc6 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: a2b9d17e-97e4-442d-985c-c4a37a9fff03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279a7bde0a3 namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: 282cf674-5c70-477e-9a40-a64c042204db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279a8330239 namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: 38de4f4e-4c43-4b89-9de5-0a5df132fc54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279a83d99a6 namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 8d9aa1a2-5b05-491a-9798-d9cea21c3951 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279ac04bd41 namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: aba12377-574b-47e3-b8e1-7eacf48af0bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279ac86d3f0 namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: c5a57c12-4835-446d-aed4-fa2b45d25dba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:00:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:36Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be3279b21c1a6a namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 08b1b43e-4edf-4b2d-93b7-adfbd7fa5e9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be328c2e206690 namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: a886f964-9bfe-44af-9a14-b8505443023d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be328c2e20f8dc namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: b114babe-e5df-43ab-b124-9fd51a9e2483 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 45fb3c58-5c38-4faa-99bf-844a1ab8a9de kind: Event lastTimestamp: "2026-07-01T15:01:21Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:21Z" name: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg.18be328c8e8047ef namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: af171245-d2dc-41b8-9cb6-cbccf39859d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7bec4-predictor-5fbd85475d namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 9c382ded-d250-4b8c-99d8-fe97d8b9edec kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: 'Created pod: error-404-isvc-7bec4-predictor-5fbd85475d-k2zzg' metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor-5fbd85475d.18be327988676ce7 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: e6003ff9-f7ed-4340-8176-64a5cf90441d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: 3b744144-0212-4dab-83a7-b58d7bddf205 kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: Scaled up replica set error-404-isvc-7bec4-predictor-5fbd85475d from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:00Z" name: error-404-isvc-7bec4-predictor.18be327986cae3d7 namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 23245e4d-02ad-4e67-bb22-83686692951d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:00:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 61819717-ceb9-45fc-90ad-ae1e6558d54c kind: Event lastTimestamp: "2026-07-01T15:00:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:00:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:30Z" name: error-404-isvc-7bec4-predictor.18be327d0595c391 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: d80564b3-1505-4654-b8e5-c17a98de3057 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:00:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 61819717-ceb9-45fc-90ad-ae1e6558d54c kind: Event lastTimestamp: "2026-07-01T15:00:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:00:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:30Z" name: error-404-isvc-7bec4-predictor.18be327d05fac190 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 519d2bd3-1840-4ee0-b504-fbc89ad156c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:00:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 61819717-ceb9-45fc-90ad-ae1e6558d54c kind: Event lastTimestamp: "2026-07-01T15:01:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:15Z" name: error-404-isvc-7bec4-predictor.18be32840391d5a7 namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: f5623340-db33-4efc-b8e9-b75dfb01b994 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:00:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 61819717-ceb9-45fc-90ad-ae1e6558d54c kind: Event lastTimestamp: "2026-07-01T15:01:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:15Z" name: error-404-isvc-7bec4-predictor.18be32840407713b namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: a2554c55-e1c4-4957-bf7f-8fb42334aee6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: c5eb1bb2-7426-4616-8d92-0e9ec1d26246 kind: Event lastTimestamp: "2026-07-01T15:00:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:46Z" name: error-404-isvc-7bec4.18be327986d34a92 namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 434003a1-9a89-4979-99f3-912a0062499d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: c5eb1bb2-7426-4616-8d92-0e9ec1d26246 kind: Event lastTimestamp: "2026-07-01T15:00:46Z" message: InferenceService [error-404-isvc-7bec4] is Ready metadata: creationTimestamp: "2026-07-01T15:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:46Z" name: error-404-isvc-7bec4.18be32846f29f82a namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 10628627-bb47-41ea-b977-7a5ea8d0d7a9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:01:20.292272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c37715bc0 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 86410fca-598e-4493-b0df-f2bde640bdea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c51961c87 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 6edb3142-b052-4a8d-bab1-98ca8fcc3684 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c5310bda7 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 3948301c-e3f8-4168-9f0c-b805fe803ed4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c55b725e2 namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: d05e31f7-7e3d-48ee-9c70-462e9adc481e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c562626db namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: ab8c9609-182b-421a-a8ad-dbeaf66907ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c56311280 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 6d2cc155-0e8b-4eec-87cf-90fc013167c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c58ec8dfc namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5484a62b-2a9d-4411-9988-8c6dfe37e96d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c5955ecde namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: 86968e59-525c-434a-bfef-3ff33b6171e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:10:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:37Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be328c6b159d82 namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: b48cea6f-7ed7-4385-acf1-eb7919557902 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be330d6c3cea8e namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: 29aff388-fa29-41fc-a768-de11eccd036c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:35Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be330d6c3e0bf6 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 9eec79d0-4d5a-414c-94d4-ba4d397ec805 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 15a428d0-0e4f-48f0-b889-ac1e37bd5ffa kind: Event lastTimestamp: "2026-07-01T15:10:37Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:37Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj.18be330ddf8da85b namespace: kserve-ci-e2e-test resourceVersion: "25873" uid: 6a37b5f8-7277-4edf-aa5e-ae82874e46c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9eaf2-predictor-86d6f679c4 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: f1fa5a93-735e-4b13-a3da-57a3d0a9d38a kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Created pod: error-404-isvc-9eaf2-predictor-86d6f679c4-lg4jj' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor-86d6f679c4.18be328c37055975 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 3b56ce20-9e22-4d8d-992a-a551acfca75a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 77b8a288-aedc-4584-af7b-e964799d5c78 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Scaled up replica set error-404-isvc-9eaf2-predictor-86d6f679c4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2-predictor.18be328c35426fa0 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 9742225d-392f-4f52-8db3-275881cdda26 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:01:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 0514125f-b022-42b2-bcb7-c6dbf90d7b0f kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-9eaf2-predictor.18be328fb3b90277 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: f3926349-c4db-4ad5-a131-e6d5f85a6e51 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:01:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 0514125f-b022-42b2-bcb7-c6dbf90d7b0f kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: error-404-isvc-9eaf2-predictor.18be328fb41bdb39 namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: 6df3ca32-482a-440f-aa09-32f6ac3482a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 0514125f-b022-42b2-bcb7-c6dbf90d7b0f kind: Event lastTimestamp: "2026-07-01T15:01:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:50Z" name: error-404-isvc-9eaf2-predictor.18be329332cc1e59 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 46786276-63b6-4011-b5af-b6bf2374878a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 0514125f-b022-42b2-bcb7-c6dbf90d7b0f kind: Event lastTimestamp: "2026-07-01T15:01:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:50Z" name: error-404-isvc-9eaf2-predictor.18be32933335edb1 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 1c24552b-68af-4767-bc0a-d21a6cfd294f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 8b34e8ed-5cc7-4ee4-a4e0-d958812350cb kind: Event lastTimestamp: "2026-07-01T15:02:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:07Z" name: error-404-isvc-9eaf2.18be328c354c348d namespace: kserve-ci-e2e-test resourceVersion: "21616" uid: 357c5055-efcf-4873-8d64-e4177de2fe76 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 8b34e8ed-5cc7-4ee4-a4e0-d958812350cb kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-9eaf2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9eaf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2.18be328c35c95772 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 4d7acfee-53d9-442c-9e7c-8df472b5b59b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 8b34e8ed-5cc7-4ee4-a4e0-d958812350cb kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9eaf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:20Z" name: error-404-isvc-9eaf2.18be328c35c9aaf7 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 6e6a01b5-f7c4-422c-b4ca-5956804d814c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: 8b34e8ed-5cc7-4ee4-a4e0-d958812350cb kind: Event lastTimestamp: "2026-07-01T15:02:07Z" message: InferenceService [error-404-isvc-9eaf2] is Ready metadata: creationTimestamp: "2026-07-01T15:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:07Z" name: error-404-isvc-9eaf2.18be329726774358 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: c35da03f-e2f6-4ef1-a229-f8f76d2a1e65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:11:45.181493Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f5851-predictor-95c7775db-wcjpb to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331db5be1f78 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 4a9daaa6-31ae-4136-987a-5e5ac95e223b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd0047310 namespace: kserve-ci-e2e-test resourceVersion: "26652" uid: 0fddc2bf-0e2a-411b-93ed-84d5de1b14db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd1cead75 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: e93161e0-21af-4165-aaf3-4d3896964c8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd4a0bd77 namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: 8ce3f026-ecbb-4406-80d1-98a67c31b0a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd51c76b6 namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: f77186a3-5e69-45b8-85bf-6e27df46bbb8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd52701c2 namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: a4b9fd61-0339-495d-9089-151b90b42928 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd8d6bd8b namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 58bba405-f848-43a8-a46c-285e11cf9c67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331dd951b366 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 97216560-b78e-4d2d-8f68-731cb3a32710 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:12:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:23Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be331e28614af2 namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: 6f0a19e0-a100-49e2-bfa5-4183e3c28933 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:29:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:14Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be34121aeca272 namespace: kserve-ci-e2e-test resourceVersion: "34236" uid: b65069a0-0c4d-4823-9b78-4b32efdd65e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:29:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:14Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be34121aed346c namespace: kserve-ci-e2e-test resourceVersion: "34237" uid: 670e564d-f12e-4084-bccf-ad2ab7b6fa1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5851-predictor-95c7775db-wcjpb namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 35e1c3d5-96d6-4d20-a44d-94df98d50628 kind: Event lastTimestamp: "2026-07-01T15:29:18Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:18Z" name: error-404-isvc-f5851-predictor-95c7775db-wcjpb.18be3412dd0ac47d namespace: kserve-ci-e2e-test resourceVersion: "34298" uid: 95ca352f-60f2-4f96-b022-bda7d8d01716 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f5851-predictor-95c7775db namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: 07fa216c-e523-43da-ae90-e0b612ea0723 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Created pod: error-404-isvc-f5851-predictor-95c7775db-wcjpb' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor-95c7775db.18be331db4fccbb3 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 612d7538-372c-485d-9a6b-88d538e89a7f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: e2642220-3941-4722-b9e0-97622c205c17 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Scaled up replica set error-404-isvc-f5851-predictor-95c7775db from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851-predictor.18be331db2c4bb49 namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 9aa5ad8f-b8be-423d-8386-f7fd4c1668e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: 03041dac-217d-4c7e-b023-bb6651d4da15 kind: Event lastTimestamp: "2026-07-01T15:12:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:15Z" name: error-404-isvc-f5851-predictor.18be3321314943aa namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: 316bf45e-df9e-490a-b737-91e612da699b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: 03041dac-217d-4c7e-b023-bb6651d4da15 kind: Event lastTimestamp: "2026-07-01T15:12:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:15Z" name: error-404-isvc-f5851-predictor.18be33213192d5cc namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: fc512c6f-a4b4-4825-9dc8-0bc073e1d735 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 03041dac-217d-4c7e-b023-bb6651d4da15 kind: Event lastTimestamp: "2026-07-01T15:13:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:00Z" name: error-404-isvc-f5851-predictor.18be33282f15ada0 namespace: kserve-ci-e2e-test resourceVersion: "27449" uid: d38f2947-31c0-4417-817f-7225fbc3204e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 03041dac-217d-4c7e-b023-bb6651d4da15 kind: Event lastTimestamp: "2026-07-01T15:13:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:00Z" name: error-404-isvc-f5851-predictor.18be33282f82411c namespace: kserve-ci-e2e-test resourceVersion: "27450" uid: 24924685-ad8f-4cd8-a598-af2363af6d8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f5851 namespace: kserve-ci-e2e-test resourceVersion: "26520" uid: 948b8c4e-390f-4e2a-b4c9-5592e5c2e2c8 kind: Event lastTimestamp: "2026-07-01T15:12:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:33Z" name: error-404-isvc-f5851.18be331db2c2d3cf namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: 596ad3c3-686f-4081-a0eb-2e6b4d4f85d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f5851 namespace: kserve-ci-e2e-test resourceVersion: "26520" uid: 948b8c4e-390f-4e2a-b4c9-5592e5c2e2c8 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-f5851": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f5851": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851.18be331db33eefcc namespace: kserve-ci-e2e-test resourceVersion: "26544" uid: eff8d11b-a1e4-4cf2-beb9-0773c1bbd9eb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f5851 namespace: kserve-ci-e2e-test resourceVersion: "26520" uid: 948b8c4e-390f-4e2a-b4c9-5592e5c2e2c8 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f5851": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:45Z" name: error-404-isvc-f5851.18be331db33f9b84 namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: cb480863-9378-43cf-89a9-d6a5f0ff4a96 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f5851 namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 948b8c4e-390f-4e2a-b4c9-5592e5c2e2c8 kind: Event lastTimestamp: "2026-07-01T15:12:33Z" message: InferenceService [error-404-isvc-f5851] is Ready metadata: creationTimestamp: "2026-07-01T15:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:33Z" name: error-404-isvc-f5851.18be3328e3cf3f95 namespace: kserve-ci-e2e-test resourceVersion: "27154" uid: 15b2cfee-c531-4c4f-b286-e0c6d58f4edd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:04.727701Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-875bcd459-l5tzn to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:04Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e0f460b95e namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: e6025ac0-0d1c-4fe2-882b-d298652625be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:04Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e10372a1e8 namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: e3519a7a-c79c-4717-bd85-68b227760eae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e1323c50c2 namespace: kserve-ci-e2e-test resourceVersion: "14693" uid: d2a6d6e9-e134-48ee-bbaa-1ef8c59b4b61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e133dfc156 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: ceef60b1-2c75-45d0-a396-208ce300111e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e136e13a26 namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: c9d2da22-a96e-40f8-a820-bb4e62e0268c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e1375c2a9c namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: b8780ef1-21de-4939-aef7-1009e3d951b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e24fe788f5 namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: a7e78e0f-0a28-4330-8934-531915fc5455 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:33Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 23.315s (23.315s including waiting). Image size: 1555870134 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e7bd98c7d5 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: f4c3d81d-6c9a-49fc-8d35-d20dc66ddbba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e7c30add46 namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9c40f70f-a5d4-4c27-80a0-10dd525aa621 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e7c40e42cb namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 4fded720-c368-4aa1-989c-715e0699e1ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:34Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e7ca1a2098 namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: bf2ff27e-5b82-45af-b8b1-6db1219d2994 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.168s (3.168s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e886ef9b44 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: e7ab37a0-130e-42db-8f63-cc93cd43aa20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e88d36e338 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: ad905fab-9e7a-4a03-8ecf-de63c8de13aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e88e4a4300 namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 0ed25a1a-c3cb-4214-b8a2-e3dc0686c6e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:50:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:44Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be31e8a4838c61 namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: 3aae470b-8d57-4d21-8cb3-25f2cfacbe4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:34Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be3203ee6bca6a namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: a4b88093-6b5a-4978-a376-a936fc474bb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:34Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be3203ee6c4ba1 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 7e3cd78c-bda7-4d7d-8d7a-6edec5fbbd4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: bd8c598b-852e-41f9-a396-d2c030df48b7 kind: Event lastTimestamp: "2026-07-01T14:51:39Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:39Z" name: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn.18be32050d2700a9 namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: f0fe97ce-9457-4df2-b0b8-5b6307c56c23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-875bcd459 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 7a87f6a1-8ce9-4dd6-9a23-7992e98cf9e1 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-875bcd459-l5tzn' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:04Z" name: isvc-sklearn-graph-1-predictor-875bcd459.18be31e0f3d99b22 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 91b234bd-1b19-4ebb-8098-ab0e3e047c80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: b75aff91-34b7-4eef-9831-05427bf1b130 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-875bcd459 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:04Z" name: isvc-sklearn-graph-1-predictor.18be31e0ebb2d7d9 namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 1caefce6-f81a-4607-8d6b-12e99be38113 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 138495b5-1f7f-4b85-bd8d-96394ca70cfa kind: Event lastTimestamp: "2026-07-01T14:50:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:04Z" name: isvc-sklearn-graph-1-predictor.18be31e4712e7417 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 884ac2a2-7aa2-46b0-a849-f66cd62c9be4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 138495b5-1f7f-4b85-bd8d-96394ca70cfa kind: Event lastTimestamp: "2026-07-01T14:50:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:04Z" name: isvc-sklearn-graph-1-predictor.18be31e471a66cb0 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 6254c407-6396-4382-9c83-6706d5856488 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:50:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 138495b5-1f7f-4b85-bd8d-96394ca70cfa kind: Event lastTimestamp: "2026-07-01T14:51:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:19Z" name: isvc-sklearn-graph-1-predictor.18be31f26d5b471c namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 3a17b623-120f-4345-883d-bf580b8938da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:50:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 138495b5-1f7f-4b85-bd8d-96394ca70cfa kind: Event lastTimestamp: "2026-07-01T14:51:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:19Z" name: isvc-sklearn-graph-1-predictor.18be31f26dc5bc17 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 8845a0c9-6fb7-470c-9a67-3f16def3c122 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 97bdbbba-9ade-4d12-b3ab-89aa2b401d02 kind: Event lastTimestamp: "2026-07-01T14:50:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:54Z" name: isvc-sklearn-graph-1.18be31e0f1d34a47 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 8988f42b-8626-40e2-be33-ffbecaa42211 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 97bdbbba-9ade-4d12-b3ab-89aa2b401d02 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:49:04Z" name: isvc-sklearn-graph-1.18be31e0f229d708 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 83b9829b-4b1f-4c19-90fb-84951d4919cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 97bdbbba-9ade-4d12-b3ab-89aa2b401d02 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:49:04Z" name: isvc-sklearn-graph-1.18be31e0f22a4a5f namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 68fb5328-8efe-46d4-b7ad-30e126f4d5f3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 97bdbbba-9ade-4d12-b3ab-89aa2b401d02 kind: Event lastTimestamp: "2026-07-01T14:50:54Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T14:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:54Z" name: isvc-sklearn-graph-1.18be31fa98926c1e namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: cac7b37d-44db-4c59-84e4-8494e31a8b43 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:05.239013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e112dac2c7 namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 4b307de5-d6c6-4214-886d-80aa87c6fdf1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e11beac1f5 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 20d570ca-50a9-4cc1-9aef-52ccf14781be reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:06Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:06Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e1506f7ade namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 4662f88d-4a60-483b-b034-cec998b11812 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:06Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" metadata: creationTimestamp: "2026-07-01T14:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:06Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e151e1d713 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: af0bfa53-86cc-4a33-8167-a21fa47acc51 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" in 13.147s (13.147s including waiting). Image size: 301875161 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e461837333 namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: bde57224-fe80-4d09-8348-e08df5e79bfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e465611e8d namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: d71bd632-c959-49d7-9f1e-941f5ef7dd86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e465d64e52 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 77a9e7f3-9647-464d-b78d-c7a066196a71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T14:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:23Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e55858f411 namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 9a7e0124-0846-43ba-a187-740cfd1e7f7c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:29Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 5.833s (5.833s including waiting). Image size: 1555870134 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:29Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6b407a100 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 520c8d87-071f-4828-9fef-482da03a0f84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:29Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6b7255522 namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: ee020864-4bbd-4f12-ba88-a1f5aea71e2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:29Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6b79e1c19 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: d2ce7b79-da30-459d-87fb-445d4a61ac86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:29Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6b7a99482 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: 7e9cbd1f-8b2f-4680-a7c3-2bff67030141 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:29Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6ba6cf5da namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 5052ede2-c5f8-48b6-a75f-776b8df5948b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:49:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:29Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6bae590ec namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 796542b7-5b3e-4df2-9bae-6197d17bed0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T14:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:51:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:40Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be31e6fbb501b1 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: 69d38613-4c26-42c1-8026-af7bd6246564 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:35Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be3203fe1a7d2a namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 0020c16f-9b6d-4f7d-b269-563c2e81f51b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:35Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be3203fe1abd14 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 5eeb3c61-99ae-4a54-98ed-d33944fa9832 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 01046ee7-828b-4082-8f82-a9fe6bfa9ebc kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s.18be3204521c97da namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 70ad599b-eaf7-4ada-ac9a-7235bc731427 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6c8478f9b5 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 10bca853-0a15-42c7-9a87-4bca3c8a14da kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6c8478f9b5-zkm5s' metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5.18be31e1126a3715 namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: fecab251-16fa-486b-a270-f8c994692450 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: b9b80d1e-ea75-4116-80d1-3e5689d367fd kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6c8478f9b5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:05Z" name: isvc-sklearn-graph-2-predictor.18be31e10f48a6a1 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 2b865c74-6fdf-45cb-ac2c-1c888a1b75f8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:49:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: b888ace2-54ec-4a12-ac8c-31eba07e8742 kind: Event lastTimestamp: "2026-07-01T14:50:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:05Z" name: isvc-sklearn-graph-2-predictor.18be31e48e2be348 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 1cd33527-05c8-4a80-ac7b-749c2bacfbf9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:49:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: b888ace2-54ec-4a12-ac8c-31eba07e8742 kind: Event lastTimestamp: "2026-07-01T14:50:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:05Z" name: isvc-sklearn-graph-2-predictor.18be31e48ea3af63 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: fa8da942-8d04-403c-9929-dcfbef778938 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:50:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: b888ace2-54ec-4a12-ac8c-31eba07e8742 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-2-predictor.18be31f2898b05a5 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 36666100-0c83-4784-8d77-60b80beb5116 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:50:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: b888ace2-54ec-4a12-ac8c-31eba07e8742 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-2-predictor.18be31f289f882ef namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 2002184e-c80d-40e3-b97a-fc2d115fd097 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 9ad9d1cb-1067-4e67-8260-eca0e8117aab kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:50Z" name: isvc-sklearn-graph-2.18be31e10f4eacd7 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: ec7cacd7-c8a2-4d4b-833d-f47ca72125c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 9ad9d1cb-1067-4e67-8260-eca0e8117aab kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:50Z" name: isvc-sklearn-graph-2.18be31f982a6c4d4 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: c2f0198e-9a6f-441e-b007-3ed528777622 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:05.121340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-nnfsp to ip-10-0-133-214.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e10bd7379d namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 6be0df73-d353-4440-aaa2-df9d2d9a57f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e125ca6072 namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: 4990c678-7883-4520-bb06-82fde28c2c70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e1274160c5 namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 241741da-06e3-451d-a396-bcfb5e6a9747 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e12a32d9d6 namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 4b92a641-3d30-4506-be3a-1c526a4f2211 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e12aac3647 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 5ac9f8bf-1846-48fa-b64c-ac2d49933621 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e24fc75578 namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: 5c5b469d-fd93-4c9c-bc09-b5c2a7bb39ee reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:34Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.6s (23.6s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e7ce79d6c3 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 8cab0040-6cd7-427f-9144-ffe5f1403f0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e7d37415db namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: ccd46875-aa58-4d76-8dc2-aa84e038112c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e7d457c413 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 19434383-0d75-4093-9a58-6c38c8bc1ba5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e7d463c5b3 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 9be57163-959c-422f-aff1-5dfe83203439 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.857s (2.857s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e87eb5410a namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: fe472d9a-bce6-47e5-a2a4-78fa1cb103a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e88b6b2b09 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 6513c44a-3adf-4751-a329-1e80c15070c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e88c9b4e8c namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: c43c8bb7-7b68-4ca7-ab65-903671b7c07f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T14:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:50:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be31e8a4350025 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 14e2b634-379a-474d-a677-dd495ad5734c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be3203fb2b5b62 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: fea8d774-17d1-4a69-a8c6-50453aca253c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be3203fb2bef61 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 87bf576f-ad7e-405e-b2c1-447bd8ba3279 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 62509ece-58c5-49f6-bdb7-771fd13e70ad kind: Event lastTimestamp: "2026-07-01T14:51:38Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-nnfsp.18be3204d1417532 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 2cec0748-9de9-4b02-a4e7-05fcb2f81056 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-214.ec2.internal source: component: kubelet host: ip-10-0-133-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: b37767df-daad-4a56-b1f7-1d334f7fed6d kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-nnfsp' metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be31e10b62ad02 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 4b1b1fa3-a394-4841-b659-e93755333da9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: f02dc32d-dae1-4caa-a748-a285a1c72758 kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:05Z" name: isvc-xgboost-graph-predictor.18be31e107e475b9 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 7071a6ad-42b0-4e45-85cf-437115b869b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3163ad19-f23d-49f9-adf4-3ed6ac0bdb96 kind: Event lastTimestamp: "2026-07-01T14:50:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:20Z" name: isvc-xgboost-graph-predictor.18be31e486661a5b namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 0183523c-e264-46a0-924b-a7074f726885 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3163ad19-f23d-49f9-adf4-3ed6ac0bdb96 kind: Event lastTimestamp: "2026-07-01T14:50:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:20Z" name: isvc-xgboost-graph-predictor.18be31e486dbbbb8 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: ee1a099e-f943-444e-b5d7-b12b8ccb601e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:50:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 3163ad19-f23d-49f9-adf4-3ed6ac0bdb96 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-predictor.18be31f6008500bb namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 46712e2b-f248-4df1-bcf5-48bc43ff3450 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:50:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 3163ad19-f23d-49f9-adf4-3ed6ac0bdb96 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-predictor.18be31f600eaa859 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 3e8331c0-6aec-4308-8f3b-ee272154de9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: b24555fe-e2c8-4aca-9dbf-9bc804ca25ad kind: Event lastTimestamp: "2026-07-01T14:50:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:43Z" name: isvc-xgboost-graph.18be31e107ea302d namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: d6a302dc-db8d-4b85-b45d-7d41d875c713 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: b24555fe-e2c8-4aca-9dbf-9bc804ca25ad kind: Event lastTimestamp: "2026-07-01T14:50:43Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T14:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:43Z" name: isvc-xgboost-graph.18be31f808f3805f namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: ae0c3f3f-dd4f-49d7-8941-697354a8220f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:51:24.675255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-6cb794ddd7-wwfvm to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T14:51:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:51:24Z" name: model-chainer-6cb794ddd7-wwfvm.18be320189e6ee47 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 6e312641-1dbc-4c66-a662-38e0ac1e76ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:24Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:51:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:24Z" name: model-chainer-6cb794ddd7-wwfvm.18be320195522239 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 17d5fe50-48bc-4a19-9143-f67c428dac19 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:25Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:51:25Z" message: Add eth0 [10.134.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:51:25Z" name: model-chainer-6cb794ddd7-wwfvm.18be3201c6e825e6 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: c16c8aff-25e0-4b29-b84d-1db8573c1f19 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:51:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:25Z" name: model-chainer-6cb794ddd7-wwfvm.18be3201c83ece08 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: cc334685-6cce-4cd0-8748-b9d04363832c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:51:25Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T14:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:25Z" name: model-chainer-6cb794ddd7-wwfvm.18be3201cbf08bbf namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: a1d4e89e-db2d-4820-aed0-60e01f2e5a7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:51:25Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T14:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:25Z" name: model-chainer-6cb794ddd7-wwfvm.18be3201cc5cbc36 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 65791384-5105-40d5-bc49-1c7eeb0f0868 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:34Z" name: model-chainer-6cb794ddd7-wwfvm.18be3203e3da26df namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: cde3c94e-9acd-49f9-8cb8-0f509eb3b46a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:51:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cb794ddd7-wwfvm namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 248317d8-01af-4a91-9c1b-70f41357f909 kind: Event lastTimestamp: "2026-07-01T14:52:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T14:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:02Z" name: model-chainer-6cb794ddd7-wwfvm.18be32047686debb namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: d82acef3-e491-4d29-802f-ed96c92210cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-6cb794ddd7 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 3e7ff255-1893-485c-8680-ebd41716ec82 kind: Event lastTimestamp: "2026-07-01T14:51:24Z" message: 'Created pod: model-chainer-6cb794ddd7-wwfvm' metadata: creationTimestamp: "2026-07-01T14:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:24Z" name: model-chainer-6cb794ddd7.18be320189828a63 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 89061885-2b66-4591-80a3-6bd62890d646 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 1648435d-3bf0-4c61-bd6e-88b538c0b8c2 kind: Event lastTimestamp: "2026-07-01T14:51:24Z" message: Scaled up replica set model-chainer-6cb794ddd7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:24Z" name: model-chainer.18be3201889ddc85 namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: fc64cb9e-2705-4105-a4f2-09dfe466079a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 135e813c-a7f3-4c45-94b4-9bdf7cb8dfa8 kind: Event lastTimestamp: "2026-07-01T14:51:24Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T14:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:24Z" name: model-chainer.18be32018dd6d2f8 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: 2327452a-09d8-4116-b7a4-3010e9c8b581 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:52:44.965213Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0d08f-7ccdc4d7bd-5tjtr to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T14:52:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:52:44Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be32143b8e7cb7 namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 7da2c990-79f9-4be8-87f1-693f22971b0a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:52:45Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be321454d56490 namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: c99a46fd-03b5-4244-babf-27bf71c71301 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0d08f} kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T14:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:45Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be321456215875 namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: a2297a58-7d01-4acb-ad7a-b85015996ac9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0d08f} kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: 'Created container: sequence-graph-0d08f' metadata: creationTimestamp: "2026-07-01T14:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:45Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be321458e9ca99 namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: e73d882f-c69c-44ac-9efc-918e2ef782e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0d08f} kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: Started container sequence-graph-0d08f metadata: creationTimestamp: "2026-07-01T14:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:45Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be3214595d8445 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 1bab01ef-508f-4567-9c8b-462c472328b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0d08f} kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Stopping container sequence-graph-0d08f metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:59Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be328768e68424 namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 7d795fc1-6a20-4cfa-9c1a-82a8a9cc0322 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0d08f} kind: Pod name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 4a8d3f1a-f9e3-4442-84cc-88787ba0f1df kind: Event lastTimestamp: "2026-07-01T15:01:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:26Z" name: sequence-graph-0d08f-7ccdc4d7bd-5tjtr.18be3287d7eaa17a namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 319899d5-bb23-4049-9276-5b7425e4e8a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0d08f-7ccdc4d7bd namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: 373d7087-1197-4a6a-858e-990a8e812fd8 kind: Event lastTimestamp: "2026-07-01T14:52:44Z" message: 'Created pod: sequence-graph-0d08f-7ccdc4d7bd-5tjtr' metadata: creationTimestamp: "2026-07-01T14:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:44Z" name: sequence-graph-0d08f-7ccdc4d7bd.18be32143b1c5522 namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: bb71ee08-7431-4014-ae52-e529b7b1b98f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0d08f namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: 2029b8f4-62fb-41ff-8fda-7c332157ce47 kind: Event lastTimestamp: "2026-07-01T14:52:44Z" message: Scaled up replica set sequence-graph-0d08f-7ccdc4d7bd from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:44Z" name: sequence-graph-0d08f.18be32143a0c0993 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 49de53bd-bc3e-4232-88f5-13bc6e210562 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0d08f namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: e4387ead-2eb8-47fc-8266-86db80069115 kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: InferenceGraph [sequence-graph-0d08f] is Ready metadata: creationTimestamp: "2026-07-01T14:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:45Z" name: sequence-graph-0d08f.18be321440380e3f namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 88fec23f-8987-4d5e-b3b8-e2983d1e2d28 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:52:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0d08f namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 99cea1dd-464c-41b0-bb88-c06bc2be33a1 kind: Event lastTimestamp: "2026-07-01T14:53:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:52:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:29Z" name: sequence-graph-0d08f.18be3217b904bede namespace: kserve-ci-e2e-test resourceVersion: "17389" uid: e7d52bd9-e0d2-45ef-9fdb-b7d587bdb46a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:52:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0d08f namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 99cea1dd-464c-41b0-bb88-c06bc2be33a1 kind: Event lastTimestamp: "2026-07-01T14:53:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:52:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:29Z" name: sequence-graph-0d08f.18be3217b96bbe78 namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: eb48020c-0db6-477c-a43a-7f6dc475162c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:02:09.834167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-102b8-558d7fd485-66cml to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:02:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:02:09Z" name: sequence-graph-102b8-558d7fd485-66cml.18be3297c05eaabb namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 2719a552-33b5-45c4-89a1-f43860d9ff73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:10Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: "2026-07-01T15:02:10Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:02:10Z" name: sequence-graph-102b8-558d7fd485-66cml.18be3297d9674018 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 20ff02d7-9fda-4ad3-9e3e-7697d70e4617 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-102b8} kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: "2026-07-01T15:02:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:10Z" name: sequence-graph-102b8-558d7fd485-66cml.18be3297daaf0b74 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 6379c256-2a20-4635-97e0-566cb545b841 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-102b8} kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: "2026-07-01T15:02:10Z" message: 'Created container: sequence-graph-102b8' metadata: creationTimestamp: "2026-07-01T15:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:10Z" name: sequence-graph-102b8-558d7fd485-66cml.18be3297dd5ecbe0 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 9204314b-aca6-4199-92ad-9e90fdb6d948 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-102b8} kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: "2026-07-01T15:02:10Z" message: Started container sequence-graph-102b8 metadata: creationTimestamp: "2026-07-01T15:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:10Z" name: sequence-graph-102b8-558d7fd485-66cml.18be3297ddd8a102 namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: b10fac78-fcb5-423d-9d11-b66228ae5c75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-102b8} kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: "2026-07-01T15:02:19Z" message: Stopping container sequence-graph-102b8 metadata: creationTimestamp: "2026-07-01T15:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:19Z" name: sequence-graph-102b8-558d7fd485-66cml.18be329a197b1455 namespace: kserve-ci-e2e-test resourceVersion: "21773" uid: adaf0192-ee4c-4a25-83f6-493fcbd1c59e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-102b8} kind: Pod name: sequence-graph-102b8-558d7fd485-66cml namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 8fbe2906-c292-47aa-80d3-83a457af7721 kind: Event lastTimestamp: "2026-07-01T15:02:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:46Z" name: sequence-graph-102b8-558d7fd485-66cml.18be329a75aeadbb namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 46168c91-ef51-4bc9-9423-8c7c26edb6d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-102b8-558d7fd485 namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: f9629458-14e1-49a0-86be-c0705fd3b5e1 kind: Event lastTimestamp: "2026-07-01T15:02:09Z" message: 'Created pod: sequence-graph-102b8-558d7fd485-66cml' metadata: creationTimestamp: "2026-07-01T15:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:09Z" name: sequence-graph-102b8-558d7fd485.18be3297bfd515a1 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 1298657d-16de-47c7-a735-3d6e5e0d36c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-102b8 namespace: kserve-ci-e2e-test resourceVersion: "21647" uid: 9117370b-e853-4a80-99a3-c3e9448771ef kind: Event lastTimestamp: "2026-07-01T15:02:09Z" message: Scaled up replica set sequence-graph-102b8-558d7fd485 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:09Z" name: sequence-graph-102b8.18be3297bf0ce269 namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: af2e22c7-a3c5-4ce5-b02c-51cd34cf95ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-102b8 namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 96b52eb1-b97b-4445-964d-7e9965e967bb kind: Event lastTimestamp: "2026-07-01T15:02:09Z" message: InferenceGraph [sequence-graph-102b8] is Ready metadata: creationTimestamp: "2026-07-01T15:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:09Z" name: sequence-graph-102b8.18be3297c2dc457e namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: a9de0e92-e894-4ce7-b040-93f6a1876e93 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:03:30.161559Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22506" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6e167-56486bcfc4-cjsnq to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:03:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:03:30Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be32aa744165cc namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 418fc04f-66e6-4afa-ac96-59429f77607d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:03:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6e167-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:03:30Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be32aa801bdf7e namespace: kserve-ci-e2e-test resourceVersion: "22522" uid: c63480dd-4909-48a2-b9d8-c6f1ec570b16 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:31Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22516" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:03:31Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:03:31Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be32aab1634248 namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: e602c8bd-9e83-470d-a2e6-713701e47c71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6e167} kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:03:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:03:31Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be32aab2ad50de namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 0b05a4b9-7194-4c3f-bcd5-a1bc2fb88f08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6e167} kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:03:31Z" message: 'Created container: sequence-graph-6e167' metadata: creationTimestamp: "2026-07-01T15:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:03:31Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be32aab5ba9800 namespace: kserve-ci-e2e-test resourceVersion: "22536" uid: aa6335d2-5a19-4c0d-97d3-51a5480de225 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6e167} kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:03:31Z" message: Started container sequence-graph-6e167 metadata: creationTimestamp: "2026-07-01T15:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:03:31Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be32aab62fff76 namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: eb2fdedc-44b6-4427-b3bf-181ebbb83ca5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6e167} kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:11:44Z" message: Stopping container sequence-graph-6e167 metadata: creationTimestamp: "2026-07-01T15:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:44Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be331da51370a4 namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: e51a1af8-c886-4c03-95c2-fc0f7dce1abc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6e167} kind: Pod name: sequence-graph-6e167-56486bcfc4-cjsnq namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a153dd8c-f1b3-40a5-bf9b-74f7266cb25e kind: Event lastTimestamp: "2026-07-01T15:12:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:12Z" name: sequence-graph-6e167-56486bcfc4-cjsnq.18be331e4ec4cfb2 namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: ba576992-6dc6-4afa-831d-3d0b998c6cf6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6e167-56486bcfc4 namespace: kserve-ci-e2e-test resourceVersion: "22503" uid: c51525c6-5b85-401a-aaec-7f57b817030d kind: Event lastTimestamp: "2026-07-01T15:03:30Z" message: 'Created pod: sequence-graph-6e167-56486bcfc4-cjsnq' metadata: creationTimestamp: "2026-07-01T15:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:30Z" name: sequence-graph-6e167-56486bcfc4.18be32aa73d76cb0 namespace: kserve-ci-e2e-test resourceVersion: "22507" uid: aa6dfa5a-ffe3-441c-b51e-958b84055251 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22501" uid: 203e65cf-fb24-4cef-ba4c-f9b7ee768a27 kind: Event lastTimestamp: "2026-07-01T15:03:30Z" message: Scaled up replica set sequence-graph-6e167-56486bcfc4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:30Z" name: sequence-graph-6e167.18be32aa72ddb3a5 namespace: kserve-ci-e2e-test resourceVersion: "22504" uid: b29a9e1a-b21b-4c72-ae35-f29ae66ad0c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22518" uid: 4924afd8-7236-4910-9caf-29e735d27ded kind: Event lastTimestamp: "2026-07-01T15:03:30Z" message: InferenceGraph [sequence-graph-6e167] is Ready metadata: creationTimestamp: "2026-07-01T15:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:03:30Z" name: sequence-graph-6e167.18be32aa78193ac6 namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: 8a6b97b6-df86-446f-92c9-648655f96310 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22502" uid: 6f33fa14-746d-4698-aab8-cc50970f1db3 kind: Event lastTimestamp: "2026-07-01T15:03:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:45Z" name: sequence-graph-6e167.18be32adf16108d8 namespace: kserve-ci-e2e-test resourceVersion: "22643" uid: 8e9c1e15-f8fc-4ef7-bf98-94db604ad82b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22502" uid: 6f33fa14-746d-4698-aab8-cc50970f1db3 kind: Event lastTimestamp: "2026-07-01T15:03:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:45Z" name: sequence-graph-6e167.18be32adf1c55f89 namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: 10dad478-f3b5-4f03-8851-6621a4d42947 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:04:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22644" uid: 6f33fa14-746d-4698-aab8-cc50970f1db3 kind: Event lastTimestamp: "2026-07-01T15:04:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:04:00Z" name: sequence-graph-6e167.18be32b170cada6d namespace: kserve-ci-e2e-test resourceVersion: "22748" uid: f55feaa7-f724-4f21-bbd7-bc8f326116ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:04:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22644" uid: 6f33fa14-746d-4698-aab8-cc50970f1db3 kind: Event lastTimestamp: "2026-07-01T15:04:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:04:00Z" name: sequence-graph-6e167.18be32b17130f78b namespace: kserve-ci-e2e-test resourceVersion: "22750" uid: c49f9ea5-6423-4e38-a456-c9cfc4f79d02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:12:55.493487Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27382" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-23f07-6646b676bd-vwjh4 to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be332e14aa0ff3 namespace: kserve-ci-e2e-test resourceVersion: "27387" uid: 9fef3853-db65-4a71-87a6-e8be04a58b7a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-23f07-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be332e21d68819 namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: 309da1be-df7a-4999-99bf-cc9522e8b8e6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:56Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27391" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:12:56Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:12:56Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be332e51d94fc1 namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: 109b87ed-9d7f-4de0-9eaa-f399475f6a67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-23f07} kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:12:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:56Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be332e532daa11 namespace: kserve-ci-e2e-test resourceVersion: "27419" uid: 3a64cdf4-c905-49ed-a1d5-4f5c5388460d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-23f07} kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:12:56Z" message: 'Created container: splitter-graph-23f07' metadata: creationTimestamp: "2026-07-01T15:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:56Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be332e5602a833 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 2aacb834-2dec-4c7c-8642-bf8664347401 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-23f07} kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:12:56Z" message: Started container splitter-graph-23f07 metadata: creationTimestamp: "2026-07-01T15:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:56Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be332e566f8701 namespace: kserve-ci-e2e-test resourceVersion: "27422" uid: 21a91d73-cba7-4d4b-9023-7f483be3ca0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-23f07} kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:21:10Z" message: Stopping container splitter-graph-23f07 metadata: creationTimestamp: "2026-07-01T15:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:10Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be33a13d4fdaf8 namespace: kserve-ci-e2e-test resourceVersion: "30864" uid: c791a206-f7d2-41c9-999d-6f54d26cc3fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:21:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-23f07} kind: Pod name: splitter-graph-23f07-6646b676bd-vwjh4 namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 0e2ad91b-fc51-41f3-bd8d-92c87b9602a7 kind: Event lastTimestamp: "2026-07-01T15:21:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:21:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:37Z" name: splitter-graph-23f07-6646b676bd-vwjh4.18be33a1d85e91c9 namespace: kserve-ci-e2e-test resourceVersion: "31096" uid: 4428cf02-37d2-4fb4-950a-d5732ad9af54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-23f07-6646b676bd namespace: kserve-ci-e2e-test resourceVersion: "27379" uid: f0f0f47c-26a2-4b5b-bd42-e96d1e35524c kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: 'Created pod: splitter-graph-23f07-6646b676bd-vwjh4' metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07-6646b676bd.18be332e144265f3 namespace: kserve-ci-e2e-test resourceVersion: "27383" uid: a1ab3faa-ef2f-44d7-81d1-573d1fa43d76 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: 33dc58e9-db85-49b9-b3cc-6439011169b6 kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: Scaled up replica set splitter-graph-23f07-6646b676bd from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07.18be332e13708ca8 namespace: kserve-ci-e2e-test resourceVersion: "27380" uid: 75f225c9-dcd1-460d-b0a4-f49b5ecd9636 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27393" uid: ebaff287-41d8-48b7-a1db-d8282030aff3 kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: InferenceGraph [splitter-graph-23f07] is Ready metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07.18be332e1833e208 namespace: kserve-ci-e2e-test resourceVersion: "27395" uid: ec6c8f09-1e86-4ac1-9f5f-9e37349a8a89 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: ebaff287-41d8-48b7-a1db-d8282030aff3 kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: 'Failed to update status for InferenceGraph "splitter-graph-23f07": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-23f07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07.18be332e1addde95 namespace: kserve-ci-e2e-test resourceVersion: "27399" uid: 61617de2-24cb-493b-a50a-3c8d4f005b0c reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: ebaff287-41d8-48b7-a1db-d8282030aff3 kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-23f07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:55Z" name: splitter-graph-23f07.18be332e1ade3250 namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: ff0ba1cd-7aad-4e3b-b353-e27770629f87 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:13:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: 7b62743e-6e52-47d6-9fba-3f50f35b6351 kind: Event lastTimestamp: "2026-07-01T15:13:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:10Z" name: splitter-graph-23f07.18be333191f7c58d namespace: kserve-ci-e2e-test resourceVersion: "27538" uid: f1a2bd37-e1ee-4548-95ce-3ca60e370396 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:13:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: 7b62743e-6e52-47d6-9fba-3f50f35b6351 kind: Event lastTimestamp: "2026-07-01T15:13:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:10Z" name: splitter-graph-23f07.18be3331925b20d4 namespace: kserve-ci-e2e-test resourceVersion: "27540" uid: 3f378b7b-4f56-4bd5-acc6-88766adaa2bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: 7b62743e-6e52-47d6-9fba-3f50f35b6351 kind: Event lastTimestamp: "2026-07-01T15:13:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:25Z" name: splitter-graph-23f07.18be33351107ffa0 namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 55ca43bd-e065-42e4-adbf-2d410ef767f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: 7b62743e-6e52-47d6-9fba-3f50f35b6351 kind: Event lastTimestamp: "2026-07-01T15:13:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:25Z" name: splitter-graph-23f07.18be3335116d6ea2 namespace: kserve-ci-e2e-test resourceVersion: "27650" uid: 7f390158-000b-4f9c-8948-962b20fe5c93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:11:35.156235Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26354" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6f42a-7bd888d8d8-dvx4g to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:11:35Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331b6030cd43 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 1aacc6ec-1cb4-44e8-805b-b01418833d72 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:35Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-6f42a-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:35Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331b6f936a05 namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 191e972b-76b1-48c1-87aa-0adee5223f3f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:36Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:11:36Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:11:36Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331b9cec325d namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: 3ff9db38-e979-4350-ad72-f80961625f51 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6f42a} kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:11:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:36Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331b9e2fd0d9 namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: cadffdaf-3cbe-4add-aae1-0117192edd03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6f42a} kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:11:36Z" message: 'Created container: splitter-graph-6f42a' metadata: creationTimestamp: "2026-07-01T15:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:36Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331ba0e6051e namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: 4f3e632c-2ee1-4e2a-abcc-a22855335d09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6f42a} kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:11:36Z" message: Started container splitter-graph-6f42a metadata: creationTimestamp: "2026-07-01T15:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:36Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331ba151de56 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: e3a5fb01-7cfe-4bd7-89f2-f39ef632e640 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6f42a} kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container splitter-graph-6f42a metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331dbbb50efd namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 52681807-8193-47b7-bbe7-ba1d08e5c59b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6f42a} kind: Pod name: splitter-graph-6f42a-7bd888d8d8-dvx4g namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: f2d452e3-5898-454d-b107-12d51ed8e570 kind: Event lastTimestamp: "2026-07-01T15:12:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:12Z" name: splitter-graph-6f42a-7bd888d8d8-dvx4g.18be331e3b134f0b namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: 53bd82b7-2e2c-45d1-945f-685a5cb8de12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6f42a-7bd888d8d8 namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: c1a910c9-d800-4152-8bb3-1fb2ca27f5cb kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: 'Created pod: splitter-graph-6f42a-7bd888d8d8-dvx4g' metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:35Z" name: splitter-graph-6f42a-7bd888d8d8.18be331b5fc1442c namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: c163759e-b9f0-4eb6-be49-c8534d20ec78 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6f42a namespace: kserve-ci-e2e-test resourceVersion: "26349" uid: 87272c48-e6d1-4453-ba2d-291284e3a5d7 kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: Scaled up replica set splitter-graph-6f42a-7bd888d8d8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:35Z" name: splitter-graph-6f42a.18be331b5ef275f7 namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: c8c74bbe-a8f2-4be7-9d35-46a6d1742048 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6f42a namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 3a28f367-054e-4a94-a43c-9d205c78219b kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: InferenceGraph [splitter-graph-6f42a] is Ready metadata: creationTimestamp: "2026-07-01T15:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:35Z" name: splitter-graph-6f42a.18be331b644d5f06 namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: 01fb4cfa-a69e-4dc3-a012-c32e29b5f6cd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:51:34.971123Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0d08f-predictor-7b6f897897-5bl7q to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be3203ef956ff8 namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 0109ce11-f77f-488a-a00d-2aa627cfe170 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0d08f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:35Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be320400066dac namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 5a154161-9363-48b8-8cc8-52f55ed892d0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be32042d2d1b82 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: 793b60f1-70d5-4632-8e60-5b2464af9eaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be32042e9f0a3f namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: b8e9bbe7-c12a-4c8a-b8f3-084dcf1e9cbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be320431713e71 namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 07592a83-876d-44ba-8e31-60ce7f1eeae0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be320431dbaf4c namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: f2dccc02-2b2d-4a01-a460-621ccce78690 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be320431e6cd8d namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: 22cde716-e878-4b66-b7b3-9a6f00968f6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be32043575d161 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 875378d1-5127-4c3d-8597-cfc0bad4a560 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T14:51:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:36Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be320435e1cd10 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 016f5fa5-3a63-44f1-8ed1-0ff0d4eb5379 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T14:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T15:01:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:02Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be320437ce0068 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: cc69618f-e00a-4159-b547-da4d20de4079 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be328770a53df3 namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: f4e042c3-2e74-4fd8-b1b3-63848ba2123d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be328770a5fcde namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: cb8b2504-6952-4e76-bb0a-d61bfa08fc0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: b323f816-6b6d-4d84-bfe0-d3f00f776296 kind: Event lastTimestamp: "2026-07-01T15:01:02Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:02Z" name: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q.18be32880045c56a namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 5a089bb7-2c7c-4497-bfb6-8633c3c9a204 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0d08f-predictor-7b6f897897 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 80b926c4-9475-49aa-b521-f4e1b4a57ede kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: 'Created pod: success-200-isvc-0d08f-predictor-7b6f897897-5bl7q' metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-0d08f-predictor-7b6f897897.18be3203eedf4c44 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 91b70d18-171a-48b0-a3eb-65ff2faba211 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: cc2f52c2-da52-4e6b-b958-ca81e2396133 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: Scaled up replica set success-200-isvc-0d08f-predictor-7b6f897897 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-0d08f-predictor.18be3203ecca35be namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: ed0f58bf-2ec7-4b21-9764-771535f01616 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:51:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 10919487-3852-4500-81e4-e5ac243fc444 kind: Event lastTimestamp: "2026-07-01T14:52:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:49Z" name: success-200-isvc-0d08f-predictor.18be32076b71d7a7 namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 8bf072c0-6086-4962-91af-a6cdca8945fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:51:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0d08f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 10919487-3852-4500-81e4-e5ac243fc444 kind: Event lastTimestamp: "2026-07-01T14:52:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:49Z" name: success-200-isvc-0d08f-predictor.18be32076bb361f5 namespace: kserve-ci-e2e-test resourceVersion: "17111" uid: 9150ded7-5b5d-4e62-87eb-3343dae1587e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: a5ce004d-9310-4d42-b64d-09ccb516d944 kind: Event lastTimestamp: "2026-07-01T14:52:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:32Z" name: success-200-isvc-0d08f.18be3203ed0341c4 namespace: kserve-ci-e2e-test resourceVersion: "16945" uid: cc3c5da4-40ee-484a-99b5-7e0b8ba85444 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: a5ce004d-9310-4d42-b64d-09ccb516d944 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-0d08f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0d08f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-0d08f.18be3203edc742b0 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: ed656e9c-8128-4e42-9629-de42a7f070a8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: a5ce004d-9310-4d42-b64d-09ccb516d944 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0d08f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-0d08f.18be3203edc79a23 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: d50f6ec3-cc51-4614-adb6-4abb2d9d3d5f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d08f namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a5ce004d-9310-4d42-b64d-09ccb516d944 kind: Event lastTimestamp: "2026-07-01T14:52:32Z" message: InferenceService [success-200-isvc-0d08f] is Ready metadata: creationTimestamp: "2026-07-01T14:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:32Z" name: success-200-isvc-0d08f.18be321147c2b108 namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 5b07ff2b-166f-40e2-8f62-b0e909b457d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:00:59.804783Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-102b8-predictor-784fbddbc5-gqzqq to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be3287724b1947 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 937cb95f-795c-401a-99b7-8d8cd89d9f5f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be32878caea229 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 1178554b-5b23-4ead-9d2d-e1d87f25498f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be32878e017832 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 430be4fe-4dfe-409b-8954-862f9c0885cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be328790d96c6d namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: cb913f93-b938-44b3-8c94-1b10f3be4da9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be3287914d3260 namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: faa78774-55e9-43c5-86cc-0e6fb994c0f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be32879158f1d9 namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 2248196a-af61-4b6c-8a91-28955866ba8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be3287959617c7 namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 3e013d33-0de5-4e23-b5ee-7cc634976a54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:00Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be3287961233fd namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: 71b3b0e4-f1c4-4e2a-826d-7f360f650fba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:01:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:46Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be3287a2ee0007 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 98a7397a-5d47-466d-9ecc-132b12ea747d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be329a2099960e namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: ee52ea4d-5b82-4c5a-ad57-9c1bae51432f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be329a20a03723 namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: c2f43e1e-3799-4dd9-a06e-d871ff5fd221 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a136e7df-7ce1-434c-a08b-a08a8fc6bba4 kind: Event lastTimestamp: "2026-07-01T15:02:21Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:21Z" name: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq.18be329a7f422b1a namespace: kserve-ci-e2e-test resourceVersion: "21948" uid: 50fc5acd-4115-4b96-8b2a-ae14127e2b3f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-102b8-predictor-784fbddbc5 namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: b682e99d-05a1-4f48-a9d4-11ff9f7db808 kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: 'Created pod: success-200-isvc-102b8-predictor-784fbddbc5-gqzqq' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-102b8-predictor-784fbddbc5.18be328771b5d981 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: a9348e6a-6c72-475e-a5cb-8c8647b50bde reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: 61908eb7-7e05-409f-8ea9-20e1eca25a7c kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: Scaled up replica set success-200-isvc-102b8-predictor-784fbddbc5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-102b8-predictor.18be32876f6419fa namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: f68849ac-369e-472d-89bc-6c302477ff67 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: d2a498e2-3e55-430a-a107-5613a274b403 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:14Z" name: success-200-isvc-102b8-predictor.18be328aedfa48a9 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 0d432eb1-ff9a-4408-b9c0-a87b762c3c9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: d2a498e2-3e55-430a-a107-5613a274b403 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:14Z" name: success-200-isvc-102b8-predictor.18be328aee5dfa13 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 097e3791-67f0-489d-973d-f60767481e42 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T15:01:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: d2a498e2-3e55-430a-a107-5613a274b403 kind: Event lastTimestamp: "2026-07-01T15:02:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:14Z" name: success-200-isvc-102b8-predictor.18be328e6d68034a namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 19ac3577-14fe-4297-8a74-8fe913e47678 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T15:01:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-102b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: d2a498e2-3e55-430a-a107-5613a274b403 kind: Event lastTimestamp: "2026-07-01T15:02:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:14Z" name: success-200-isvc-102b8-predictor.18be328e6dd03f6d namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: 50ebe505-7c03-4ad3-8b50-1f140315449c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: c39873a7-f50e-4c6c-a44e-6f56ecc7f07c kind: Event lastTimestamp: "2026-07-01T15:01:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:56Z" name: success-200-isvc-102b8.18be32876f681c58 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: 1f7e762b-b75b-441b-b893-6b48d2ddc06b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: c39873a7-f50e-4c6c-a44e-6f56ecc7f07c kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-102b8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-102b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-102b8.18be3287703abfd5 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: e1611514-3605-4477-956a-7656830fd18a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: c39873a7-f50e-4c6c-a44e-6f56ecc7f07c kind: Event lastTimestamp: "2026-07-01T15:00:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-102b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:59Z" name: success-200-isvc-102b8.18be3287703b19b2 namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: c2764ea6-4659-4c10-b57e-e4c57e54f7ed reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-102b8 namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: c39873a7-f50e-4c6c-a44e-6f56ecc7f07c kind: Event lastTimestamp: "2026-07-01T15:01:56Z" message: InferenceService [success-200-isvc-102b8] is Ready metadata: creationTimestamp: "2026-07-01T15:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:56Z" name: success-200-isvc-102b8.18be3294b24ab172 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 367c172f-1623-4f03-b7f6-7d0cf2dc900c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:11:45.583174Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-23f07-predictor-56689dfcb5-clmch to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331dcdaf5d43 namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: ae0283c9-ed1b-42ad-ad90-0cbb74fe4cd0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-23f07-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331ddbbb728f namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: 3147797c-43b9-4743-85bc-834676d2f2f5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e0b6040fc namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: 6e9d89f9-4f40-4f9b-82d1-2a11c9ad16bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e0cca52be namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b9c425d3-d417-47b8-a9de-8bb120436f04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e0fea873f namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 058a1a71-8f64-4efe-a7ad-cb0a9f2764fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e106c50a2 namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: fe168a6d-95da-4c06-bc9e-4f9cb3fd95aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e10786e0a namespace: kserve-ci-e2e-test resourceVersion: "26727" uid: ce613944-cfce-4eaa-8dc6-41ca8af9e2ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e134220d4 namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: bebd7e5b-8e97-4947-a9af-e1fce133bcd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e13b9d12d namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: e1ea6c5e-b37d-4b08-b603-8c16b9991736 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:12:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:33Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be331e407eb272 namespace: kserve-ci-e2e-test resourceVersion: "27161" uid: 0580fdf5-c14d-44d8-a4fd-37b49c854ee7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:21:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:10Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be33a145a508b0 namespace: kserve-ci-e2e-test resourceVersion: "30881" uid: 4a569b78-a976-4134-aae6-8de5f2a16c25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23f07-predictor-56689dfcb5-clmch namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 29cf3e83-f0cf-45bc-ba30-fab178b651e4 kind: Event lastTimestamp: "2026-07-01T15:21:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:21:10Z" name: success-200-isvc-23f07-predictor-56689dfcb5-clmch.18be33a145a5da69 namespace: kserve-ci-e2e-test resourceVersion: "30882" uid: 7931fb43-9dfc-4631-acc4-5ca86ee2ff8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-23f07-predictor-56689dfcb5 namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: 97476178-4753-4538-a2d6-9c4fa852f39c kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Created pod: success-200-isvc-23f07-predictor-56689dfcb5-clmch' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-23f07-predictor-56689dfcb5.18be331dccecda51 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 9d9163dc-0a5e-4d15-94dd-afadf90d4489 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 3bb83cd5-6012-4503-9d18-56379febb726 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Scaled up replica set success-200-isvc-23f07-predictor-56689dfcb5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-23f07-predictor.18be331dcb3b55a6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: d95d0ccf-c10f-4e80-ba3b-298b1880e38d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: b3c628f1-0380-41c7-9c0f-846e6bf28879 kind: Event lastTimestamp: "2026-07-01T15:12:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:30Z" name: success-200-isvc-23f07-predictor.18be332149a13232 namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: a625b6e1-212a-411a-a227-6810b96e6efa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: b3c628f1-0380-41c7-9c0f-846e6bf28879 kind: Event lastTimestamp: "2026-07-01T15:12:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:30Z" name: success-200-isvc-23f07-predictor.18be332149ff6660 namespace: kserve-ci-e2e-test resourceVersion: "27122" uid: 99b573fe-79a3-4076-bf3e-bf9387f57e80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: b3c628f1-0380-41c7-9c0f-846e6bf28879 kind: Event lastTimestamp: "2026-07-01T15:13:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:15Z" name: success-200-isvc-23f07-predictor.18be332bc656c211 namespace: kserve-ci-e2e-test resourceVersion: "27582" uid: 32ac0d8f-3351-422d-a729-7fbbab833a7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: b3c628f1-0380-41c7-9c0f-846e6bf28879 kind: Event lastTimestamp: "2026-07-01T15:13:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:15Z" name: success-200-isvc-23f07-predictor.18be332bc6c1f7c9 namespace: kserve-ci-e2e-test resourceVersion: "27583" uid: 3643713d-340f-4d7c-b24a-98388b1a26e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23f07 namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 02ce1d37-74f8-4875-b446-f539ae033fa8 kind: Event lastTimestamp: "2026-07-01T15:12:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:43Z" name: success-200-isvc-23f07.18be331dcb38c31c namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: 132bf5e5-a9cd-4454-a385-b26b6fe3e20b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23f07 namespace: kserve-ci-e2e-test resourceVersion: "27245" uid: 02ce1d37-74f8-4875-b446-f539ae033fa8 kind: Event lastTimestamp: "2026-07-01T15:12:43Z" message: InferenceService [success-200-isvc-23f07] is Ready metadata: creationTimestamp: "2026-07-01T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:43Z" name: success-200-isvc-23f07.18be332b4fb299d7 namespace: kserve-ci-e2e-test resourceVersion: "27246" uid: ff43de65-5ded-4bd6-a80d-3adb46c88196 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:50:44.972606Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3297d-predictor-5494fbf467-jmcgg to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:50:44Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f84b709f15 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 806b4bcf-0028-4267-b08d-144467c4d6b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3297d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f8551cdf22 namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 12fd60dc-5cba-4596-a071-fc0bfd8ac08d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f8894b7154 namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 519c6cd0-d930-4fe8-9990-61e3304e1337 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f88ac73099 namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: c5ac3303-f00b-44f3-896a-fbc1287d7c51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f88d79219a namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: e4b60827-7170-454e-bba7-c65752edd0bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f88de31504 namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 72841d47-ecd9-4262-98b2-9eca8f22eb70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f88dee45ca namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 1d29ea71-5972-4b23-a421-b7df4bf233f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f890b60c5a namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: 79bda33a-ef8d-4050-92bd-cd450c5ede99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:50:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:46Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f8911af103 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 07e9b2a7-6104-4e97-86f7-98c6a42abfba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T14:50:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T15:00:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:03Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be31f8fd7624ab namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 445add1c-47c6-4bd5-a7b7-203ea2da0522 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be327983ba35d5 namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: 2d520441-cd8d-4742-8ddf-bc1a48ac4124 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be327983be478c namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: b8bf2748-b5fa-4b83-9f5f-7143b996991c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 124c2e52-d9a7-48ce-86e5-0e24d7c801b1 kind: Event lastTimestamp: "2026-07-01T15:00:03Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:03Z" name: success-200-isvc-3297d-predictor-5494fbf467-jmcgg.18be327a71efc933 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 42f0eabf-b11f-44c5-b7af-02939c0391e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3297d-predictor-5494fbf467 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 1dfce6d5-2105-4b91-b08f-7301b0724e53 kind: Event lastTimestamp: "2026-07-01T14:50:44Z" message: 'Created pod: success-200-isvc-3297d-predictor-5494fbf467-jmcgg' metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:44Z" name: success-200-isvc-3297d-predictor-5494fbf467.18be31f84ad14b32 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 6a782bab-4b3f-42a4-acbf-fd77dfddfd17 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 728ea1bd-dbc1-4aec-94c4-45cabc7257ef kind: Event lastTimestamp: "2026-07-01T14:50:44Z" message: Scaled up replica set success-200-isvc-3297d-predictor-5494fbf467 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:44Z" name: success-200-isvc-3297d-predictor.18be31f8478f12c9 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 2706be6d-2a38-45e6-8c5d-057fcc5c057b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:50:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: f866dc1f-e47f-40cb-972d-0c5d3d0f8e72 kind: Event lastTimestamp: "2026-07-01T14:51:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:14Z" name: success-200-isvc-3297d-predictor.18be31fbc64b3bb6 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3b5efcb5-8f64-470e-b6a6-f7d871fc7cbc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:50:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: f866dc1f-e47f-40cb-972d-0c5d3d0f8e72 kind: Event lastTimestamp: "2026-07-01T14:51:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:14Z" name: success-200-isvc-3297d-predictor.18be31fbc6b15f00 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 23df932d-0791-425f-8f17-90d7ef09204a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:51:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: f866dc1f-e47f-40cb-972d-0c5d3d0f8e72 kind: Event lastTimestamp: "2026-07-01T14:52:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:15Z" name: success-200-isvc-3297d-predictor.18be3202c4a9a3e9 namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: bdd2052b-dbf7-4c2b-af3a-2622968cc6b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:51:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3297d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: f866dc1f-e47f-40cb-972d-0c5d3d0f8e72 kind: Event lastTimestamp: "2026-07-01T14:52:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:15Z" name: success-200-isvc-3297d-predictor.18be3202c5148a71 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: d104c497-9522-4472-9c49-fb16849a1fa2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 1f4a33ac-572f-4468-be40-b63b31fcbd22 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-3297d.18be31f8479538f9 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 432832bd-b4e7-46d0-8740-e6437a8ab311 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 1f4a33ac-572f-4468-be40-b63b31fcbd22 kind: Event lastTimestamp: "2026-07-01T14:50:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-3297d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3297d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:44Z" name: success-200-isvc-3297d.18be31f84816b38b namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 3de2e308-d2ce-4003-a553-9b00c5ca9c23 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 1f4a33ac-572f-4468-be40-b63b31fcbd22 kind: Event lastTimestamp: "2026-07-01T14:50:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3297d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:44Z" name: success-200-isvc-3297d.18be31f848170921 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 7630744e-1862-4a5c-b571-5ede53f4e99e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3297d namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 1f4a33ac-572f-4468-be40-b63b31fcbd22 kind: Event lastTimestamp: "2026-07-01T14:51:34Z" message: InferenceService [success-200-isvc-3297d] is Ready metadata: creationTimestamp: "2026-07-01T14:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:34Z" name: success-200-isvc-3297d.18be3203b953527f namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 00441ae5-c7ba-4c2f-bb31-0e2cc4ccbe2f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:04.842202Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:04Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e0fb33e6f9 namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: 0110c99d-dfc6-4b18-aefc-07fe943b71ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-680c6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:04Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e103da931b namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 10077bbb-a136-489b-86df-a6344ffabcce reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:05Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e1390caa03 namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: 13e5f332-5a12-412a-b606-0bb991142d4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:05Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:05Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e13a81b087 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 5b11cf1d-ff89-4c13-981e-440b5a0f81dd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" in 13.608s (13.608s including waiting). Image size: 1335614579 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e4659cabe0 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 704d96e3-29e4-4a3a-ad19-bc38692d0fea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e469309cd0 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 7782d691-acff-4c63-9a50-f4aed499f04b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e469afbce7 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 2b760baf-2aa7-4dce-8673-afc4863ac432 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:19Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e469be4cfb namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 0907f878-4d35-4d00-8a95-eb41b5d9eac5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.455s (2.455s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:22Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e4fc217e12 namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: ab839167-8b9f-4d5f-b532-2b440b4c72b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:22Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e4feec6627 namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 8f550cfd-5ce3-4f3b-b8c6-2d901a388db2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:22Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e4ff51c324 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 2973832a-df13-42fc-8d93-990d64c62d74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:49:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:49:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:58Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31e51c713f2f namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: b5d051c2-9976-4386-9c28-229d3164beca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31f854718ba8 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: bad13d62-b429-41e2-b4b5-b8bd8fb0e628 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:50:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:45Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31f85475e914 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 1d2e86d6-24de-45dd-897d-f1e5ac2242c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 0d2bfcc6-62bf-4850-b9fd-30ac703445bc kind: Event lastTimestamp: "2026-07-01T14:50:49Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-01T14:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:49Z" name: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g.18be31f95e4f0f21 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 576cb36d-8f9e-4716-846e-67332b4bb605 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-680c6-predictor-7d48c66ccf namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 7ecb4aed-7c66-4c7f-9ac3-bdad8a1ef2c6 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: 'Created pod: success-200-isvc-680c6-predictor-7d48c66ccf-x8h4g' metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:04Z" name: success-200-isvc-680c6-predictor-7d48c66ccf.18be31e0fa44ef4f namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 604841e0-6991-4d1c-a5c4-45e9fc7ed74c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 08f3c514-381d-47c4-bcb7-082919c8c4b9 kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: Scaled up replica set success-200-isvc-680c6-predictor-7d48c66ccf from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:04Z" name: success-200-isvc-680c6-predictor.18be31e0f82ef5f7 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: f99d8495-b1f8-4c6f-8d38-a6c59523011a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 8207c0fa-db8b-4c06-9265-4759e30914a4 kind: Event lastTimestamp: "2026-07-01T14:49:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:49Z" name: success-200-isvc-680c6-predictor.18be31e476b3abdc namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 5028969b-1960-4d0a-bb1c-ce06196b213f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 8207c0fa-db8b-4c06-9265-4759e30914a4 kind: Event lastTimestamp: "2026-07-01T14:49:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:49Z" name: success-200-isvc-680c6-predictor.18be31e47710cca5 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 2916d0af-00e9-485d-bbde-80607eee296d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 8207c0fa-db8b-4c06-9265-4759e30914a4 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:34Z" name: success-200-isvc-680c6-predictor.18be31eef334237d namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 8e139176-532b-4b34-8d0c-3de1e6832682 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-680c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 8207c0fa-db8b-4c06-9265-4759e30914a4 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:34Z" name: success-200-isvc-680c6-predictor.18be31eef39c7c74 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 725ee08a-4ea6-4af0-8a50-d9e4ff50d799 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-680c6 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: b366a493-514b-4174-8fa2-eb5979fec710 kind: Event lastTimestamp: "2026-07-01T14:50:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:08Z" name: success-200-isvc-680c6.18be31e0f834d7cb namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: c66949ce-9d51-4c7f-b738-72ec0ee43d7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-680c6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: b366a493-514b-4174-8fa2-eb5979fec710 kind: Event lastTimestamp: "2026-07-01T14:50:08Z" message: InferenceService [success-200-isvc-680c6] is Ready metadata: creationTimestamp: "2026-07-01T14:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:08Z" name: success-200-isvc-680c6.18be31efd98e16f8 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 89839251-f47b-46bf-8fe9-d7e90dfed9c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:02:20.065563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6e167-predictor-765674cfdc-r74gk to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a2235c699 namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 92803c3a-ea07-49cd-9595-3102da418228 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a3d03c4c2 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 0897fee3-e5ed-445e-aa7e-80d3951076e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a3e4db966 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 03a1a150-4c87-4f77-95ce-02c596340842 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a4177b56f namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: 7e3b6fcf-bdca-486c-8126-3974a489b334 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a41f5493e namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 44aa2450-b395-41a7-a50e-ecbcf2012831 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a42005ad7 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 8d201ed4-cd19-4e4c-99a8-07ece7e79695 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a466a7334 namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: ad95378c-92cc-4a22-bea4-d7f245e9dc01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a46e65970 namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 175ee6cb-ae66-44f1-a122-4a92c019c6fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T15:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:11:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:47Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be329a59ee2a1f namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: 275e7c3f-4f2f-4dca-9d4e-88962321d5bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be331dae478e14 namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 7d2c287f-a3a9-4993-a5e8-4563d4313136 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be331dae4821c7 namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 96f65475-6aac-4002-9b79-93c4cd7830bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e167-predictor-765674cfdc-r74gk namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: e184d628-16a0-4d10-a4d4-eecfeb8b19a4 kind: Event lastTimestamp: "2026-07-01T15:11:47Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:47Z" name: success-200-isvc-6e167-predictor-765674cfdc-r74gk.18be331e2262e1fd namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 13c08cf5-8776-49a7-87ce-5cf2f03a74d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6e167-predictor-765674cfdc namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 6c1c4b28-a3f5-4118-affe-69ee44e3c32b kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Created pod: success-200-isvc-6e167-predictor-765674cfdc-r74gk' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor-765674cfdc.18be329a21b877ae namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: e3212af9-450c-4083-b1e7-8279934cb13c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 21fcddfc-12af-42bb-a24e-fe78b19ad030 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: Scaled up replica set success-200-isvc-6e167-predictor-765674cfdc from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167-predictor.18be329a1ffae531 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: f7f101bb-9b76-4051-b974-30c446768319 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:02:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 1896eb79-d5da-4c99-8277-51e50e8d6a7b kind: Event lastTimestamp: "2026-07-01T15:02:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:50Z" name: success-200-isvc-6e167-predictor.18be329d9edfdcbb namespace: kserve-ci-e2e-test resourceVersion: "22185" uid: 8c4de9e2-2426-4c49-ada9-85e5b7a648d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:02:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 1896eb79-d5da-4c99-8277-51e50e8d6a7b kind: Event lastTimestamp: "2026-07-01T15:02:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:50Z" name: success-200-isvc-6e167-predictor.18be329d9f3eecc3 namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 4c1a0664-9019-4db6-b43f-321d9254c19e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:03:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 1896eb79-d5da-4c99-8277-51e50e8d6a7b kind: Event lastTimestamp: "2026-07-01T15:03:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:35Z" name: success-200-isvc-6e167-predictor.18be32a49d4ef869 namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 5139ec04-ccb5-4239-a4e7-34bd02cd3fa5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:03:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e167-predictor namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 1896eb79-d5da-4c99-8277-51e50e8d6a7b kind: Event lastTimestamp: "2026-07-01T15:03:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:03:35Z" name: success-200-isvc-6e167-predictor.18be32a49db69b89 namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: 43876a92-05fc-4faf-a357-810a64a090a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: d1bfd30d-8a7b-428c-a162-17e35e86422c kind: Event lastTimestamp: "2026-07-01T15:03:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:03:17Z" name: success-200-isvc-6e167.18be329a20331239 namespace: kserve-ci-e2e-test resourceVersion: "22406" uid: 9e36977b-c058-4617-9fcc-163611cdf79d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: d1bfd30d-8a7b-428c-a162-17e35e86422c kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'Failed to update status for InferenceService "success-200-isvc-6e167": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6e167": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167.18be329a209ca94b namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: c21ed229-2fe7-49f7-a14a-d2a7faf9c110 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: d1bfd30d-8a7b-428c-a162-17e35e86422c kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6e167": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-6e167.18be329a209d1f52 namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: 2f6a7346-60de-43a6-977e-24e14b0f329f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:03:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e167 namespace: kserve-ci-e2e-test resourceVersion: "22404" uid: d1bfd30d-8a7b-428c-a162-17e35e86422c kind: Event lastTimestamp: "2026-07-01T15:03:17Z" message: InferenceService [success-200-isvc-6e167] is Ready metadata: creationTimestamp: "2026-07-01T15:03:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:03:17Z" name: success-200-isvc-6e167.18be32a76b735d42 namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: ccb27cef-0c74-43a8-aed1-38235cd14146 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:10:35.115082Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6f42a-predictor-bc5969967-ft9tk to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330d657598ea namespace: kserve-ci-e2e-test resourceVersion: "25783" uid: 32ceec73-0c0d-4629-a80a-f9375c42fa60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6f42a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330d70565fa8 namespace: kserve-ci-e2e-test resourceVersion: "25835" uid: 826ea76c-d3d5-45b5-9d48-68f6bb2a7d95 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330da3396db5 namespace: kserve-ci-e2e-test resourceVersion: "25848" uid: 32993a67-d36d-4aee-9290-6585d38a51ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330da4baebab namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: cbc86fca-bf4f-4f09-9acb-2872933d7db7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330da7b837f1 namespace: kserve-ci-e2e-test resourceVersion: "25854" uid: 461763d2-cabb-4be4-949b-d11db68ddc48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330da831b921 namespace: kserve-ci-e2e-test resourceVersion: "25855" uid: fbbdf0fe-4ad6-466c-832a-d8f9c5d4693e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330da83c72e1 namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: 1ba59a59-5bb0-4abe-a152-18ec4f7f163d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330dabd8cbe1 namespace: kserve-ci-e2e-test resourceVersion: "25858" uid: c3a5a2b1-6744-45f6-9bbb-cfdb05dfc8c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:10:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:36Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330dac57d8f8 namespace: kserve-ci-e2e-test resourceVersion: "25860" uid: b8a0bec0-52c4-41b7-843d-75b5f5d71ee5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:11:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:13Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be330de18ba1ce namespace: kserve-ci-e2e-test resourceVersion: "26164" uid: 8f08f3c2-fc69-48e1-a265-a4a07ee72e1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be331dc2bf7af6 namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: 5632b9e0-c20d-457f-9b73-352fe32272ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be331dc2c006d2 namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: 55826357-03de-4d04-9541-a62a53abf785 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: aab1fdfa-2add-4de3-8545-7f304858df06 kind: Event lastTimestamp: "2026-07-01T15:11:48Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:48Z" name: success-200-isvc-6f42a-predictor-bc5969967-ft9tk.18be331e69b9e3a9 namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: 857a07dc-4c0d-4be8-9246-84841e87d9fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6f42a-predictor-bc5969967 namespace: kserve-ci-e2e-test resourceVersion: "25762" uid: 47360e3e-ffd4-4be7-81a5-6516ccd57d2a kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'Created pod: success-200-isvc-6f42a-predictor-bc5969967-ft9tk' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-6f42a-predictor-bc5969967.18be330d650534ae namespace: kserve-ci-e2e-test resourceVersion: "25777" uid: 108d5331-31cb-48a0-bfc2-f503e2cf5047 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: 31e7ab2f-3779-4f42-b66d-ac3363a2038f kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: Scaled up replica set success-200-isvc-6f42a-predictor-bc5969967 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-6f42a-predictor.18be330d617e935d namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: be1a5f9b-dbb3-420c-907e-fa08a9747228 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:10:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: b90a23bf-db0e-4fbd-afdd-88b75e5445ef kind: Event lastTimestamp: "2026-07-01T15:11:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:05Z" name: success-200-isvc-6f42a-predictor.18be3310dfbbeed3 namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: ed525ca3-f149-4414-b073-189ce76cdea2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:10:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: b90a23bf-db0e-4fbd-afdd-88b75e5445ef kind: Event lastTimestamp: "2026-07-01T15:11:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:05Z" name: success-200-isvc-6f42a-predictor.18be3310e0236327 namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: daeea0d0-2be7-42b3-b738-c9df38cae058 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: b90a23bf-db0e-4fbd-afdd-88b75e5445ef kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:35Z" name: success-200-isvc-6f42a-predictor.18be3317dd95a24e namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: d54fa6aa-f81a-47bb-87f5-33d7ff698cc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6f42a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: b90a23bf-db0e-4fbd-afdd-88b75e5445ef kind: Event lastTimestamp: "2026-07-01T15:11:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:35Z" name: success-200-isvc-6f42a-predictor.18be3317ddf8f16e namespace: kserve-ci-e2e-test resourceVersion: "26345" uid: 0596dac5-1d46-40f2-be10-1c551868ef06 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 2e423bd8-5a2e-4e49-a745-a241e23a2b3f kind: Event lastTimestamp: "2026-07-01T15:11:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:23Z" name: success-200-isvc-6f42a.18be330d617cc520 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: fc05de84-0a3d-4a56-933b-1c42f03d3eed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 2e423bd8-5a2e-4e49-a745-a241e23a2b3f kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-6f42a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6f42a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-6f42a.18be330d620c0aa8 namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 7e1a8dab-76ae-4aee-bcbf-a4535f8d24f3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 2e423bd8-5a2e-4e49-a745-a241e23a2b3f kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6f42a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-6f42a.18be330d620c7e11 namespace: kserve-ci-e2e-test resourceVersion: "25770" uid: acbff7e4-00b5-478c-b589-406862cc958f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6f42a namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 2e423bd8-5a2e-4e49-a745-a241e23a2b3f kind: Event lastTimestamp: "2026-07-01T15:11:23Z" message: InferenceService [success-200-isvc-6f42a] is Ready metadata: creationTimestamp: "2026-07-01T15:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:11:23Z" name: success-200-isvc-6f42a.18be33189cda6938 namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 1cecc1a4-de1c-4a71-8d81-396e2c5afc65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:59:59.966460Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be327983a6dfbb namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 566fc048-54b2-47df-baca-ee1bf553b231 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7bec4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:00Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be32798e5c5f37 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: f59050a4-b59c-47c4-ac3f-66b6905ce3fe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279c1c0016c namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: bfdfca9a-3ae3-43c0-afe8-25b72c19e8bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279c32d23dc namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: a29975c8-9cc4-427d-b7ff-88f42fc12bd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279c678828c namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 1c6302fb-94c6-461c-95fb-9a3f75ddf8ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279c6eff350 namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: ad8a735a-3d34-474b-a9b8-8b9c89cd900e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279c6fae3a3 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: b0660c89-1d37-4bdb-9275-9ed1f8d56019 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279c9e44741 namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: c7be8779-b37d-4556-85c5-3f7d5688ce10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:01Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279ca5ed03e namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: 7fb03fe4-abb0-47fa-b937-2f8250ea3f8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:00:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:47Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be3279d736b8ef namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 719f18fc-d148-4281-bcda-47e36dde0f8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be328c31960d72 namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 0652287a-8a7a-4c54-97f7-4e30920e14ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be328c3197afd6 namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: dcf7e815-5daf-4c58-87fc-dcc3d41bfb24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: be6143cc-5b12-440c-9a8b-d562b6f874bf kind: Event lastTimestamp: "2026-07-01T15:01:22Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:22Z" name: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7.18be328cb360a3c6 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 829e442c-2f5b-434d-9414-5d585ebc6517 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7bec4-predictor-7d98f4b667 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: e87d339b-a166-45dc-9eca-46750f39f531 kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: 'Created pod: success-200-isvc-7bec4-predictor-7d98f4b667-58sb7' metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-7bec4-predictor-7d98f4b667.18be32798334c4c8 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 952ef575-4212-4c3a-9339-fc4c26c49644 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 2a0b69c0-21cd-4f76-ab5c-74cbd6ec6cd5 kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: Scaled up replica set success-200-isvc-7bec4-predictor-7d98f4b667 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-7bec4-predictor.18be32797fa809bb namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: aa695454-9b8a-4e53-8f75-90909020ae6d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:00:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: b9806fdd-4e3a-49bc-9141-d54fb8fb1b81 kind: Event lastTimestamp: "2026-07-01T15:00:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:00:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:44Z" name: success-200-isvc-7bec4-predictor.18be327cfe8b8abe namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 444aee0b-4fd0-4552-8b78-8bc18c11907f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:00:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: b9806fdd-4e3a-49bc-9141-d54fb8fb1b81 kind: Event lastTimestamp: "2026-07-01T15:00:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:00:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:44Z" name: success-200-isvc-7bec4-predictor.18be327cfef8f7e5 namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 614e4f41-dc09-4059-965e-301a9a8d75fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: b9806fdd-4e3a-49bc-9141-d54fb8fb1b81 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:14Z" name: success-200-isvc-7bec4-predictor.18be32877b9fcd27 namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 00b53886-5d26-486c-af78-c37002e4cc40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7bec4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: b9806fdd-4e3a-49bc-9141-d54fb8fb1b81 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:14Z" name: success-200-isvc-7bec4-predictor.18be32877c023808 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 3f47dbce-1485-4fb5-9f42-2d73ebcca6e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: b5da388a-640c-46a1-ada7-392550beb870 kind: Event lastTimestamp: "2026-07-01T15:00:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:57Z" name: success-200-isvc-7bec4.18be32797fda5bc6 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: f5cbca77-839e-47ae-84b1-035a0449b0d0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: b5da388a-640c-46a1-ada7-392550beb870 kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-7bec4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7bec4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-7bec4.18be32798033667b namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 54ef3b51-a1ea-4253-a79b-d9ba5d86ee90 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: b5da388a-640c-46a1-ada7-392550beb870 kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7bec4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:59Z" name: success-200-isvc-7bec4.18be32798033b227 namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 8db4bf9f-45f2-4d34-b7a6-66b564d8226c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7bec4 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: b5da388a-640c-46a1-ada7-392550beb870 kind: Event lastTimestamp: "2026-07-01T15:00:57Z" message: InferenceService [success-200-isvc-7bec4] is Ready metadata: creationTimestamp: "2026-07-01T15:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:57Z" name: success-200-isvc-7bec4.18be3286e64eb03b namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: b04c434a-35e5-48d0-83bc-b680650617f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:01:20.195751Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c31b08a0a namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: 0384a94e-7d8f-4999-92ff-e519eaa0a142 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c4cad24ff namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: e72e15e7-20aa-4dd8-9817-081d48827b56 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c4e39cf6e namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 3960dc1d-f070-4108-bffb-ef8a59b888c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c51601d71 namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 588ae328-b2cf-4220-9879-0c0e06e812da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c51d97f19 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 17dfccc9-a256-4d15-89d2-34ebc72c38f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c51e58de3 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: d92909b5-0085-46a7-b860-42dc77b03b8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c54adbae0 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 8320648c-65c8-477b-b173-8f652a3dfb75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c55c2b235 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: d08a72e5-d6c2-47b2-967d-afea5a96465c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:10:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:37Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be328c8c96a7a8 namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: 2b8788dd-8fe7-4f4d-bcd5-57743ea391bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be330d627176df namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: 49baf7c4-c295-4f33-bbf1-731a103a2f23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:10:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:35Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be330d6272151c namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: 158969af-940e-4987-a532-3449af0918b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 3f561b4d-7097-4caf-988c-d5176f698299 kind: Event lastTimestamp: "2026-07-01T15:10:37Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:10:37Z" name: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn.18be330e0100b182 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: c9d50de4-4361-41bd-8cdc-12fb6a4af14d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9eaf2-predictor-65cbb969f namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 43d00a1c-a256-4b43-afb5-13ffb09f4b41 kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: 'Created pod: success-200-isvc-9eaf2-predictor-65cbb969f-rqxxn' metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor-65cbb969f.18be328c30fcf84f namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: fa3d80de-1b3a-4729-93db-269dd1335dc4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: ce19214c-9b81-4541-8fc9-5b0b18d7536f kind: Event lastTimestamp: "2026-07-01T15:01:20Z" message: Scaled up replica set success-200-isvc-9eaf2-predictor-65cbb969f from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:20Z" name: success-200-isvc-9eaf2-predictor.18be328c2ef7f69a namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: f6bb1cc1-daad-4c06-85cb-d02dd1abfe88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:01:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 199da1db-5b74-4dfe-927e-271db50397c7 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-9eaf2-predictor.18be328fad6a3dbe namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: dc071fbf-bb23-420b-a872-d7116a5e82ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:01:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 199da1db-5b74-4dfe-927e-271db50397c7 kind: Event lastTimestamp: "2026-07-01T15:02:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:20Z" name: success-200-isvc-9eaf2-predictor.18be328fadcdd93c namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 41955035-665b-4e17-a8fc-bce24cdabcf5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 199da1db-5b74-4dfe-927e-271db50397c7 kind: Event lastTimestamp: "2026-07-01T15:01:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:50Z" name: success-200-isvc-9eaf2-predictor.18be32932c88f6cc namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: dc356ab5-e813-4b42-82db-497ad3faef94 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9eaf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 199da1db-5b74-4dfe-927e-271db50397c7 kind: Event lastTimestamp: "2026-07-01T15:01:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:50Z" name: success-200-isvc-9eaf2-predictor.18be32932cfbd04d namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: e80e1539-bd1c-4eec-9f93-3da848be4e5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T15:01:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 9762f539-40c0-4456-8c9d-55ac13bd62e2 kind: Event lastTimestamp: "2026-07-01T15:02:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:07Z" name: success-200-isvc-9eaf2.18be328c2efa97d4 namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: be11065b-26f8-4b0b-bf78-8b2e1f49bea1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9eaf2 namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: 9762f539-40c0-4456-8c9d-55ac13bd62e2 kind: Event lastTimestamp: "2026-07-01T15:02:07Z" message: InferenceService [success-200-isvc-9eaf2] is Ready metadata: creationTimestamp: "2026-07-01T15:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:07Z" name: success-200-isvc-9eaf2.18be3297479bb8c9 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: 7b58c285-c98c-4f6c-9101-26ea55a43c69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:11:45.057644Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f5851-predictor-856647c799-rjkbq to ip-10-0-139-55.ec2.internal metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331dae5c6d84 namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 6fc90e9a-2516-4164-befc-aaee63ff0544 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f5851-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331dbda265f7 namespace: kserve-ci-e2e-test resourceVersion: "26581" uid: 2845fee7-4638-49a5-ab59-534d016521fe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331decf7a4be namespace: kserve-ci-e2e-test resourceVersion: "26692" uid: f904a394-c2da-44e3-a184-06b3576098ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331df03edc4f namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: d500db03-16f7-44fc-9cd7-25222af66539 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331df3eda7c9 namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: f86499ab-5d11-4aba-9071-b3d68946416c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331df45e2be5 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 77aa9b11-0dbb-4b6a-bfdf-9ee75f100bdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331df46980cd namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 47514f47-7eb3-4bf7-84c7-04222795bf64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331df93bc59e namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: 013cf7ec-a723-4416-8d4c-5614e12aae93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:11:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:11:46Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331df9aa7829 namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: e0cf7d7c-a264-4633-8488-801b4ee3cbc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:12:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:32Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be331e045f74c7 namespace: kserve-ci-e2e-test resourceVersion: "27137" uid: 0111a0fb-ff28-45d1-92e3-e189ddbe2519 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:29:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:14Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be341215e748ff namespace: kserve-ci-e2e-test resourceVersion: "34221" uid: 806464ba-1902-42f8-99e7-18e69eeca9c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:29:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:14Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be341215e7e5a4 namespace: kserve-ci-e2e-test resourceVersion: "34222" uid: ec08aa95-80a6-47f6-94cb-27e33247b741 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5851-predictor-856647c799-rjkbq namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 3a289d83-2171-4993-86a9-d38df8649fa0 kind: Event lastTimestamp: "2026-07-01T15:29:17Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:17Z" name: success-200-isvc-f5851-predictor-856647c799-rjkbq.18be3412b9543d92 namespace: kserve-ci-e2e-test resourceVersion: "34294" uid: a62dfd7a-be5a-44db-924d-de0c778b2bb4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-55.ec2.internal source: component: kubelet host: ip-10-0-139-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f5851-predictor-856647c799 namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 2566dcf5-c7ce-49b4-9355-12a3991994dd kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: 'Created pod: success-200-isvc-f5851-predictor-856647c799-rjkbq' metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-f5851-predictor-856647c799.18be331dadef0251 namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: a48d96b8-725c-4ce4-8301-9a01bbbef517 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: c4a08b86-b5a2-4fa5-b9fc-fb4ddb5214f5 kind: Event lastTimestamp: "2026-07-01T15:11:45Z" message: Scaled up replica set success-200-isvc-f5851-predictor-856647c799 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:11:45Z" name: success-200-isvc-f5851-predictor.18be331dabb28c54 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 01a5b37a-8b5c-4927-aae7-ca11635c4535 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 155c091a-e070-4736-851d-d5b84a98fb0d kind: Event lastTimestamp: "2026-07-01T15:12:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:30Z" name: success-200-isvc-f5851-predictor.18be33212a5cedd7 namespace: kserve-ci-e2e-test resourceVersion: "27114" uid: 7f615f21-d414-4149-ab51-e827769726bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 155c091a-e070-4736-851d-d5b84a98fb0d kind: Event lastTimestamp: "2026-07-01T15:12:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:30Z" name: success-200-isvc-f5851-predictor.18be33212ac4533e namespace: kserve-ci-e2e-test resourceVersion: "27115" uid: 9f9211a0-7f91-4110-9946-e07b1ce44ccc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 155c091a-e070-4736-851d-d5b84a98fb0d kind: Event lastTimestamp: "2026-07-01T15:13:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:15Z" name: success-200-isvc-f5851-predictor.18be332ba79a0f1e namespace: kserve-ci-e2e-test resourceVersion: "27573" uid: 082ac09b-3098-4fa9-8f87-73d792148b20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T15:12:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5851-predictor namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 155c091a-e070-4736-851d-d5b84a98fb0d kind: Event lastTimestamp: "2026-07-01T15:13:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:15Z" name: success-200-isvc-f5851-predictor.18be332ba8034bf0 namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: 82571877-33ed-450c-b86d-69852a05b82a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f5851 namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: e1cb4709-2eac-4e03-a527-5453864a4f82 kind: Event lastTimestamp: "2026-07-01T15:12:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:42Z" name: success-200-isvc-f5851.18be331dabb47ff7 namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 2bc49b8e-9b73-4697-82d3-fe650d9bb0fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f5851 namespace: kserve-ci-e2e-test resourceVersion: "27231" uid: e1cb4709-2eac-4e03-a527-5453864a4f82 kind: Event lastTimestamp: "2026-07-01T15:12:42Z" message: InferenceService [success-200-isvc-f5851] is Ready metadata: creationTimestamp: "2026-07-01T15:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:42Z" name: success-200-isvc-f5851.18be332b148f941a namespace: kserve-ci-e2e-test resourceVersion: "27232" uid: e1f9a62a-5a10-4cbb-8c52-6e93d6a1da96 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:51:44.988643Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3297d-b7f46cb6-kp7sg to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T14:51:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:51:44Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be320644ac9877 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: fbeba697-e28e-4a69-a861-1b14680cbec2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:45Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T14:51:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3297d-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:45Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be320652e2913d namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 845dd1f8-74c9-4c4d-bee0-6691b1bd0636 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T14:51:46Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:51:46Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be320681d36427 namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 8e5a1a76-5e04-46d9-9417-0de89a2174c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3297d} kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T14:51:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:46Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be3206831ff234 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: fa691cd8-a682-4ad8-8663-cd491b19602c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3297d} kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T14:51:46Z" message: 'Created container: switch-graph-3297d' metadata: creationTimestamp: "2026-07-01T14:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:46Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be320686bad8a8 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 7444a76f-69c0-468e-830c-956e5858639e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3297d} kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T14:51:46Z" message: Started container switch-graph-3297d metadata: creationTimestamp: "2026-07-01T14:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:46Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be32068730a5e8 namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: cd1355b1-4750-4435-902b-e8e1dc00e1fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3297d} kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T14:59:59Z" message: Stopping container switch-graph-3297d metadata: creationTimestamp: "2026-07-01T14:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:59Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be327976dd88be namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: c2907047-6a75-4bc0-9317-9279304be317 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:00:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3297d} kind: Pod name: switch-graph-3297d-b7f46cb6-kp7sg namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 41178cfc-8fa5-4573-8d03-55fc29266f0a kind: Event lastTimestamp: "2026-07-01T15:00:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:00:27Z" name: switch-graph-3297d-b7f46cb6-kp7sg.18be327a0edde50b namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 14856693-b8e4-4faf-a965-56e168529c2f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3297d-b7f46cb6 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 3ca0e755-ef9f-4d90-b8eb-4d15f64d7fc6 kind: Event lastTimestamp: "2026-07-01T14:51:44Z" message: 'Created pod: switch-graph-3297d-b7f46cb6-kp7sg' metadata: creationTimestamp: "2026-07-01T14:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:44Z" name: switch-graph-3297d-b7f46cb6.18be32064419e981 namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: 8cee66c4-6935-47ec-8f82-c6669a398860 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3297d namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 113b80e1-5c6a-4989-bd58-40f34b1b0b4b kind: Event lastTimestamp: "2026-07-01T14:51:44Z" message: Scaled up replica set switch-graph-3297d-b7f46cb6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:44Z" name: switch-graph-3297d.18be3206433a3491 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 0f650b40-de7a-4566-9746-03a548283faf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3297d namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 47425f17-7f34-476b-b0e4-c1a50673fdd5 kind: Event lastTimestamp: "2026-07-01T14:51:45Z" message: InferenceGraph [switch-graph-3297d] is Ready metadata: creationTimestamp: "2026-07-01T14:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:45Z" name: switch-graph-3297d.18be320649225894 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: da46f6d4-f489-46aa-be43-b91c69b141d1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:51:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3297d namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: c9b1f46a-8df8-4cce-a723-664ea89ff75a kind: Event lastTimestamp: "2026-07-01T14:52:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:30Z" name: switch-graph-3297d.18be3209c21e0fb5 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 6ba2b5bd-9a2b-4856-9937-4f6aac789045 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:51:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3297d namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: c9b1f46a-8df8-4cce-a723-664ea89ff75a kind: Event lastTimestamp: "2026-07-01T14:52:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:30Z" name: switch-graph-3297d.18be3209c285b537 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: e481f6ed-d7b0-432c-b360-4424ed49c304 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:50:34.633495Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-680c6-bd8865647-tqtbt to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T14:50:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:50:34Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f5e32e42f6 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 5e03f7cb-366f-4b85-bce4-548e3005434a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:50:35Z" message: Add eth0 [10.134.0.15/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:50:35Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f5fc8ee91e namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 46706826-4194-46b7-acd6-645c6f11f57b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-680c6} kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:50:35Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" metadata: creationTimestamp: "2026-07-01T14:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:35Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f5fdd19599 namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: 4f00c965-08d6-4ad4-829c-390534db2f8e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-680c6} kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:50:37Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" in 2.365s (2.365s including waiting). Image size: 218070141 bytes.' metadata: creationTimestamp: "2026-07-01T14:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:37Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f68ad47a5a namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 1a9e5e08-92b1-446c-8b35-ac1229abfc00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-680c6} kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:50:37Z" message: 'Created container: switch-graph-680c6' metadata: creationTimestamp: "2026-07-01T14:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:37Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f68e23f4db namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 37b7706e-e1d1-4bfc-80fe-1e10a81d7a8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-680c6} kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:50:37Z" message: Started container switch-graph-680c6 metadata: creationTimestamp: "2026-07-01T14:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:37Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f68e934103 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 228a5f5d-2b3c-42f5-bac3-efb44ca6513c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-680c6} kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:50:44Z" message: Stopping container switch-graph-680c6 metadata: creationTimestamp: "2026-07-01T14:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:44Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f83e59aaf3 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: aa87ca2a-750a-4689-a815-bd4e7270f576 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-680c6} kind: Pod name: switch-graph-680c6-bd8865647-tqtbt namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: ffbdf5f0-44e0-4fbb-8d97-f092861dfd5f kind: Event lastTimestamp: "2026-07-01T14:51:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T14:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:14Z" name: switch-graph-680c6-bd8865647-tqtbt.18be31f93f895666 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 3beb340b-bad3-4e22-8a07-4a563e24a411 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-680c6-bd8865647 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 8d46cd9b-4271-4e16-8f8c-944abb4c6bb5 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'Created pod: switch-graph-680c6-bd8865647-tqtbt' metadata: creationTimestamp: "2026-07-01T14:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:34Z" name: switch-graph-680c6-bd8865647.18be31f5e2b6cf10 namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 37ffbc42-7274-4a3f-be5d-d625e2dccdd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-680c6 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 4292cab6-56ca-4046-a388-afe52f485ca9 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: Scaled up replica set switch-graph-680c6-bd8865647 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:34Z" name: switch-graph-680c6.18be31f5e1f0f4f1 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: f570a7ac-4f5c-4ff5-97f8-9fd8a1916471 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-680c6 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: bcd15046-aa5c-4d19-bbbd-5b3b99ddc7c0 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: InferenceGraph [switch-graph-680c6] is Ready metadata: creationTimestamp: "2026-07-01T14:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:34Z" name: switch-graph-680c6.18be31f5e5b19282 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: d7fbb157-3389-4521-879f-cda4ef97075e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-680c6 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: bcd15046-aa5c-4d19-bbbd-5b3b99ddc7c0 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'Failed to update status for InferenceGraph "switch-graph-680c6": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-680c6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:34Z" name: switch-graph-680c6.18be31f5e85722cb namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 2658ea20-9075-47ab-8f60-bc47ff0d1a23 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-680c6 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: bcd15046-aa5c-4d19-bbbd-5b3b99ddc7c0 kind: Event lastTimestamp: "2026-07-01T14:50:34Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-680c6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:34Z" name: switch-graph-680c6.18be31f5e8578b48 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: ac8f36f2-41be-4f23-ad5c-83568b5b2422 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:12:55.124269Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f5851-d97c7f4c4-vrgc7 to ip-10-0-137-172.ec2.internal metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be332dfea82d80 namespace: kserve-ci-e2e-test resourceVersion: "27357" uid: 4bc8b48f-46ec-4ae9-a2af-102a673763a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-99c95bd8b-vkw49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be332e18684515 namespace: kserve-ci-e2e-test resourceVersion: "27396" uid: 8ab1509d-47e1-4abe-893e-79cc5a15a343 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f5851} kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be332e19a4e828 namespace: kserve-ci-e2e-test resourceVersion: "27398" uid: 8a8ab44a-464c-45f1-aee4-facf59ac33d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f5851} kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: 'Created container: switch-graph-f5851' metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be332e1c3d3e65 namespace: kserve-ci-e2e-test resourceVersion: "27401" uid: a812f063-7c64-4971-89cd-a24407d137e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f5851} kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: Started container switch-graph-f5851 metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be332e1cb11988 namespace: kserve-ci-e2e-test resourceVersion: "27403" uid: 1a02c0d7-756f-43b1-9292-21781da62683 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:29:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f5851} kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: "2026-07-01T15:29:14Z" message: Stopping container switch-graph-f5851 metadata: creationTimestamp: "2026-07-01T15:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:14Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be34120a71c9bc namespace: kserve-ci-e2e-test resourceVersion: "34203" uid: 9a871ed9-c404-4d54-994e-d11f000afa7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T15:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f5851} kind: Pod name: switch-graph-f5851-d97c7f4c4-vrgc7 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: f5bab8ac-98bb-469e-98ac-b0777b0b737d kind: Event lastTimestamp: "2026-07-01T15:29:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T15:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:29:41Z" name: switch-graph-f5851-d97c7f4c4-vrgc7.18be341288cb3e93 namespace: kserve-ci-e2e-test resourceVersion: "34472" uid: 076e16a1-315a-4e0c-8e59-22c56ef18815 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-172.ec2.internal source: component: kubelet host: ip-10-0-137-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f5851-d97c7f4c4 namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: c9326dc8-d883-46be-8183-0d97ccbf7cab kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: 'Created pod: switch-graph-f5851-d97c7f4c4-vrgc7' metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851-d97c7f4c4.18be332dfe133b43 namespace: kserve-ci-e2e-test resourceVersion: "27353" uid: 1b0d9f4b-675c-4898-bfdf-2c9126a1b257 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f5851 namespace: kserve-ci-e2e-test resourceVersion: "27347" uid: 4d4750a6-2049-41ec-9332-b0e24212f51e kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: Scaled up replica set switch-graph-f5851-d97c7f4c4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851.18be332dfd289061 namespace: kserve-ci-e2e-test resourceVersion: "27350" uid: c50ba810-c365-4057-99fd-87e1fd8d1708 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:12:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f5851 namespace: kserve-ci-e2e-test resourceVersion: "27364" uid: fd591b5c-cc2d-418b-b575-2e5f2f8ef233 kind: Event lastTimestamp: "2026-07-01T15:12:55Z" message: InferenceGraph [switch-graph-f5851] is Ready metadata: creationTimestamp: "2026-07-01T15:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:12:55Z" name: switch-graph-f5851.18be332e03088692 namespace: kserve-ci-e2e-test resourceVersion: "27366" uid: 5bd5754b-b871-43eb-aadd-937c8d912c1e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:13:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f5851 namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: d8d86f42-8fa0-4ea1-8d63-cb0ab2beae12 kind: Event lastTimestamp: "2026-07-01T15:13:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:25Z" name: switch-graph-f5851.18be33317bb41ae2 namespace: kserve-ci-e2e-test resourceVersion: "27642" uid: 32cc8422-30d7-428e-bef2-dcc23e440658 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:13:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f5851 namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: d8d86f42-8fa0-4ea1-8d63-cb0ab2beae12 kind: Event lastTimestamp: "2026-07-01T15:13:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:13:25Z" name: switch-graph-f5851.18be33317c0d76af namespace: kserve-ci-e2e-test resourceVersion: "27643" uid: 4acd8e61-e12c-48f7-b412-2543763d1596 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34490"